cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Query utils: Use webmercator utils, reuse code and always substitute tokens

Browse Source
This commit is contained in:
Raul Marin 2019-07-15 14:16:21 +02:00
parent bdbe132311
commit 63b6af2ac7
1 changed files with 42 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
lib/cartodb/utils/query-utils.js
View File

@ -1,48 +1,41 @@
'use strict';
const SubstitutionTokens = require('./substitution-tokens');
const windshaftUtils = require('windshaft').utils;
function prepareQuery(sql) {
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
}
module.exports.extractTableNames = function extractTableNames(query) {
module.exports.extractTableNames = function (query) {
return [
'SELECT * FROM CDB_QueryTablesText($windshaft$',
prepareQuery(query),
substituteDummyTokens(query),
'$windshaft$) as tablenames'
].join('');
};
module.exports.getQueryActualRowCount = function (query) {
return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`;
return `select COUNT(*) AS rows FROM (${substituteDummyTokens(query)}) AS __cdb_query`;
};
function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows';
return 'select CDB_EstimateRowCount($windshaft$' + substituteDummyTokens(query) + '$windshaft$) as rows';
}
module.exports.getQueryRowEstimation = getQueryRowEstimation;
function getQueryGeometryType(query, geometryColumn) {
return `
SELECT ST_GeometryType(${geometryColumn}) AS geom_type
FROM (${substituteDummyTokens(query)}) AS __cdb_query
WHERE ${geometryColumn} IS NOT NULL
LIMIT 1
`;
}
module.exports.getQueryGeometryType = getQueryGeometryType;
module.exports.getAggregationMetadata = ctx => `
WITH
rowEstimation AS (
${getQueryRowEstimation(ctx.query)}
),
geometryType AS (
SELECT ST_GeometryType(${ctx.geometryColumn}) as geom_type
FROM (${ctx.query}) AS __cdb_query WHERE ${ctx.geometryColumn} IS NOT NULL LIMIT 1
${getQueryGeometryType(ctx.query, ctx.geometryColumn)}
)
SELECT
rows AS count,
@ -85,7 +78,7 @@ module.exports.getQueryTopCategories = function (query, column, topN, includeNul
const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`;
return `
SELECT ${column} AS category, COUNT(*) AS frequency
FROM (${query}) AS __cdb_query
FROM (${substituteDummyTokens(query)}) AS __cdb_query
${where}
GROUP BY ${column} ORDER BY 2 DESC
LIMIT ${topN}
@ -116,7 +109,7 @@ module.exports.getQuerySample = function (query, sampleProb, limit = null, rando
SELECT setseed(${randomSeed})
)
SELECT ${columnSelector(columns)}
FROM (${query}) AS __cdb_query
FROM (${substituteDummyTokens(query)}) AS __cdb_query
WHERE random() < ${sampleProb}
${limitClause}
`;
@ -157,19 +150,11 @@ function simpleQueryTable(sql) {
return false;
}
module.exports.getQueryGeometryType = function (query, geometryColumn) {
return `
SELECT ST_GeometryType(${geometryColumn}) AS geom_type
FROM (${query}) AS __cdb_query
WHERE ${geometryColumn} IS NOT NULL
LIMIT 1
`;
};
function getQueryLimited(query, limit = 0) {
return `
SELECT *
FROM (${query}) AS __cdb_query
FROM (${substituteDummyTokens(query)}) AS __cdb_query
LIMIT ${limit}
`;
}
@ -181,32 +166,32 @@ function queryPromise(dbConnection, query) {
}
function substituteDummyTokens(sql) {
return sql && SubstitutionTokens.replace(sql, {
bbox: 'ST_MakeEnvelope(0,0,0,0)',
scale_denominator: '0',
pixel_width: '1',
pixel_height: '1'
});
return subsituteTokensForZoom(sql, 0);
}
function subsituteTokensForZoom(sql, zoom, singleTile=false) {
const tileRes = 256;
const wmSize = 6378137.0*2*Math.PI;
const nTiles = Math.pow(2, zoom);
const tileSize = wmSize / nTiles;
const resolution = tileSize / tileRes;
const scaleDenominator = resolution / 0.00028;
const x0 = -wmSize/2, y0 = -wmSize/2;
let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`;
if (singleTile) {
bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`;
function subsituteTokensForZoom(sql, zoom) {
if (!sql) {
return undefined;
}
return SubstitutionTokens.replace(sql, {
bbox: bbox,
scale_denominator: scaleDenominator,
pixel_width: resolution,
pixel_height: resolution
});
const affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
const webmercator = new windshaftUtils.WebMercatorHelper();
const resolution = webmercator.getResolution({ z : zoom });
const scaleDenominator = resolution.dividedBy(0.00028);
// We always use the whole world as the bbox
const extent = webmercator.getExtent({ x : 0, y : 0, z : 0 });
return sql
.replace(affectedTableRegexCache.bbox,
`ST_MakeEnvelope(${extent.xmin}, ${extent.ymin}, ${extent.xmax}, ${extent.ymax}, 3857)`)
.replace(affectedTableRegexCache.scale_denominator, scaleDenominator)
.replace(affectedTableRegexCache.pixel_width, resolution)
.replace(affectedTableRegexCache.pixel_height, resolution);
}
module.exports.queryPromise = queryPromise;