Merge branch 'master' into turbo-carto-category

This commit is contained in:
Raul Ochoa 2016-05-19 13:43:35 +02:00
commit 3bacfecc49
24 changed files with 1359 additions and 74 deletions

20
NEWS.md
View File

@ -1,9 +1,27 @@
# Changelog
## 2.42.3
## 2.44.0
Released 2016-mm-dd
Adds support for sql wrap in all layers
## 2.43.1
Released 2016-05-19
Bug fixes:
- Dataview error when bbox present without query rewrite data #458
## 2.43.0
Released 2016-05-18
New features:
- Overviews now support dataviews and filtering #449
## 2.42.2

View File

@ -0,0 +1,59 @@
var _ = require('underscore');
var step = require('step');
var CamshaftFilter = require('../models/filter/camshaft');
function FilterStatsApi(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
module.exports = FilterStatsApi;
function getEstimatedRows(pgQueryRunner, username, query, callback) {
pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) {
if (err){
callback(err);
return;
}
var rows;
if ( result_rows[0] && result_rows[0]['QUERY PLAN'] &&
result_rows[0]['QUERY PLAN'][0] && result_rows[0]['QUERY PLAN'][0].Plan ) {
rows = result_rows[0]['QUERY PLAN'][0].Plan['Plan Rows'];
}
return callback(null, rows);
});
}
FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) {
var stats = {};
var self = this;
step(
function getUnfilteredRows() {
getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this);
},
function receiveUnfilteredRows(err, rows) {
if (err){
callback(err);
return;
}
stats.unfiltered_rows = rows;
this(null, rows);
},
function getFilteredRows() {
if ( filters && !_.isEmpty(filters)) {
var camshaftFilter = new CamshaftFilter(filters);
var query = camshaftFilter.sql(unfiltered_query);
getEstimatedRows(self.pgQueryRunner, username, query, this);
} else {
this(null, null);
}
},
function receiveFilteredRows(err, rows) {
if (err){
callback(err);
return;
}
stats.filtered_rows = rows;
callback(null, stats);
}
);
};

View File

@ -8,7 +8,13 @@ var step = require('step');
var Timer = require('../stats/timer');
var BBoxFilter = require('../models/filter/bbox');
var DataviewFactory = require('../models/dataview/factory');
var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
});
function DataviewBackend(analysisBackend) {
this.analysisBackend = analysisBackend;
@ -16,7 +22,6 @@ function DataviewBackend(analysisBackend) {
module.exports = DataviewBackend;
DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
var self = this;
@ -101,6 +106,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
ownFilter = !!ownFilter;
var query;
if (ownFilter) {
query = node.getQuery();
} else {
@ -109,11 +115,48 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
query = node.getQuery(applyFilters);
}
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(
mapConfig.obj().layers,
function(l){ return l.options.source && (l.options.source.id === sourceId); }
);
var queryRewriteData = layer && layer.options.query_rewrite_data;
if ( queryRewriteData ) {
if ( node.type === 'source' ) {
var filters = node.filters; // TODO: node.getFilters() when available in camshaft
var filters_disabler = Object.keys(filters).reduce(
function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
{}
);
var unfiltered_query = node.getQuery(filters_disabler);
queryRewriteData = _.extend(
{},
queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query }
);
}
}
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
query = bboxFilter.sql(query);
if ( queryRewriteData ) {
var bbox_filter_definition = {
type: 'bbox',
options: {
column: 'the_geom',
srid: 4326,
},
params: {
bbox: params.bbox
}
};
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
}
}
var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
);
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
function castNumbers(overrides, val, k) {
@ -123,7 +166,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
{ownFilter: ownFilter}
);
var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
dataview.getResult(pg, overrideParams, this);
},
function returnCallback(err, result) {
@ -277,4 +320,4 @@ function dbParamsFromReqParams(params) {
dbParams.dbname = params.dbname;
}
return dbParams;
}
}

View File

@ -19,6 +19,7 @@ var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adap
var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
var SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
/**
* @param {AuthApi} authApi
@ -52,6 +53,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend);
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
this.overviewsAdapter = overviewsAdapter;
this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter();
}
util.inherits(MapController, BaseController);
@ -139,6 +141,10 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
self.req2params(req, this);
},
prepareConfigFn,
function prepareSqlWrap(err, requestMapConfig) {
assert.ifError(err);
self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this);
},
function prepareAnalysisLayers(err, requestMapConfig) {
assert.ifError(err);
var analysisConfiguration = {
@ -185,17 +191,20 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
function addOverviewsInformation(err, requestMapConfig, datasource) {
assert.ifError(err);
var next = this;
self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) {
if (err) {
return next(err);
}
self.overviewsAdapter.getLayers(
req.context.user, requestMapConfig.layers, analysesResults,
function(err, layers) {
if (err) {
return next(err);
}
if (layers) {
requestMapConfig.layers = layers;
}
if (layers) {
requestMapConfig.layers = layers;
}
return next(null, requestMapConfig, datasource);
});
return next(null, requestMapConfig, datasource);
}
);
},
function parseTurboCarto(err, requestMapConfig, datasource) {
assert.ifError(err);

View File

@ -0,0 +1,32 @@
var parentFactory = require('../factory');
var dataviews = require('./');
function OverviewsDataviewFactory(queryRewriter, queryRewriteData, options) {
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
this.options = options;
}
OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinition) {
var type = dataviewDefinition.type;
var dataviews = OverviewsDataviewMetaFactory.dataviews;
if ( !this.queryRewriter || !this.queryRewriteData || !dataviews[type] ) {
return parentFactory.getDataview(query, dataviewDefinition);
}
return new dataviews[type](
query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
);
};
var OverviewsDataviewMetaFactory = {
dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
return allDataviews;
}, {}),
getFactory: function(queryRewriter, queryRewriteData, options) {
return new OverviewsDataviewFactory(queryRewriter, queryRewriteData, options);
},
};
module.exports = OverviewsDataviewMetaFactory;

View File

@ -0,0 +1,104 @@
var _ = require('underscore');
var BaseWidget = require('../base');
var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overviews');
var dot = require('dot');
dot.templateSettings.strip = false;
var formulaQueryTpls = {
'count': dot.template([
'SELECT',
'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
};
function Formula(query, options, queryRewriter, queryRewriteData, params) {
this.base_dataview = new BaseDataview(query, options);
this.query = query;
this.column = options.column || '1';
this.operation = options.operation;
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
this.options = params;
}
Formula.prototype = new BaseWidget();
Formula.prototype.constructor = Formula;
module.exports = Formula;
var zoom_level_factor = 100.0;
// Compute zoom level so that the the resolution grid size of the
// selected overview is smaller (zoom_level_factor times smaller at least)
// than the bounding box size.
function zoom_level_for_bbox(bbox) {
var px_per_tile = 256.0;
var earth_width = 360.0;
// TODO: now we assume overviews are computed for 1-pixel tolerance;
// should use extended overviews metadata to compute this properly.
if ( bbox ) {
var bbox_values = _.map(bbox.split(','), function(v) { return +v; });
var w = Math.abs(bbox_values[2]-bbox_values[0]);
var h = Math.abs(bbox_values[3]-bbox_values[1]);
var max_dim = Math.min(w, h);
// Find minimum suitable z
// note that the QueryRewirter will use the minimum level overview
// of level >= z if it exists, and otherwise the base table
var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0));
return Math.max(z, 0);
}
return 0;
}
Formula.prototype.sql = function(psql, filters, override, callback) {
var _query = this.query;
var formulaQueryTpl = formulaQueryTpls[this.operation];
if ( formulaQueryTpl ) {
// supported formula for use with overviews
var zoom_level = zoom_level_for_bbox(this.options.bbox);
_query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level });
var formulaSql = formulaQueryTpl({
_query: _query,
_operation: this.operation,
_column: this.column
});
debug(formulaSql);
callback = callback || override;
return callback(null, formulaSql);
}
// For non supported operations (min, max) we're not using overviews.
return this.base_dataview.sql(psql, filters, override, callback);
};
Formula.prototype.format = function(result) {
return this.base_dataview.format(result);
};
Formula.prototype.getType = function() {
return this.base_dataview.getType();
};
Formula.prototype.toString = function() {
return this.base_dataview.toString();
};

View File

@ -0,0 +1,3 @@
module.exports = {
Formula: require('./formula')
};

View File

@ -0,0 +1,35 @@
var filters = {
category: require('./camshaft/category'),
range: require('./camshaft/range')
};
function createFilter(filterDefinition) {
var filterType = filterDefinition.type.toLowerCase();
if (!filters.hasOwnProperty(filterType)) {
throw new Error('Unknown filter type: ' + filterType);
}
return new filters[filterType](filterDefinition.column, filterDefinition.params);
}
function CamshaftFilters(filters) {
this.filters = filters;
}
CamshaftFilters.prototype.sql = function(rawSql) {
var filters = this.filters || {};
var applyFilters = {};
return Object.keys(filters)
.filter(function(filterName) {
return applyFilters.hasOwnProperty(filterName) ? applyFilters[filterName] : true;
})
.map(function(filterName) {
var filterDefinition = filters[filterName];
return createFilter(filterDefinition);
})
.reduce(function(sql, filter) {
return filter.sql(sql);
}, rawSql);
};
module.exports = CamshaftFilters;

View File

@ -0,0 +1,79 @@
'use strict';
var debug = require('debug')('windshaft:filter:category');
var dot = require('dot');
dot.templateSettings.strip = false;
var filterQueryTpl = dot.template([
'SELECT *',
'FROM ({{=it._sql}}) _camshaft_category_filter',
'WHERE {{=it._filters}}'
].join('\n'));
var escapeStringTpl = dot.template('$escape_{{=it._i}}${{=it._value}}$escape_{{=it._i}}$');
var inConditionTpl = dot.template('{{=it._column}} IN ({{=it._values}})');
var notInConditionTpl = dot.template('{{=it._column}} NOT IN ({{=it._values}})');
function Category(column, filterParams) {
this.column = column;
if (!Array.isArray(filterParams.accept) && !Array.isArray(filterParams.reject)) {
throw new Error('Category filter expects at least one array in accept or reject params');
}
if (Array.isArray(filterParams.accept) && Array.isArray(filterParams.reject)) {
if (filterParams.accept.length === 0 && filterParams.reject.length === 0) {
throw new Error(
'Category filter expects one value either in accept or reject params when both are provided'
);
}
}
this.accept = filterParams.accept;
this.reject = filterParams.reject;
}
module.exports = Category;
/*
- accept: [] => reject all
- reject: [] => accept all
*/
Category.prototype.sql = function(rawSql) {
var valueFilters = [];
if (Array.isArray(this.accept)) {
if (this.accept.length > 0) {
valueFilters.push(inConditionTpl({
_column: this.column,
_values: this.accept.map(function(value, i) {
return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value});
}).join(',')
}));
} else {
valueFilters.push('0 = 1');
}
}
if (Array.isArray(this.reject)) {
if (this.reject.length > 0) {
valueFilters.push(notInConditionTpl({
_column: this.column,
_values: this.reject.map(function (value, i) {
return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value});
}).join(',')
}));
} else {
valueFilters.push('1 = 1');
}
}
debug(filterQueryTpl({
_sql: rawSql,
_filters: valueFilters.join(' AND ')
}));
return filterQueryTpl({
_sql: rawSql,
_filters: valueFilters.join(' AND ')
});
};

View File

@ -0,0 +1,43 @@
'use strict';
var dot = require('dot');
dot.templateSettings.strip = false;
var betweenFilterTpl = dot.template('{{=it._column}} BETWEEN {{=it._min}} AND {{=it._max}}');
var minFilterTpl = dot.template('{{=it._column}} >= {{=it._min}}');
var maxFilterTpl = dot.template('{{=it._column}} <= {{=it._max}}');
var filterQueryTpl = dot.template('SELECT * FROM ({{=it._sql}}) _camshaft_range_filter WHERE {{=it._filter}}');
function Range(column, filterParams) {
this.column = column;
if (!Number.isFinite(filterParams.min) && !Number.isFinite(filterParams.max)) {
throw new Error('Range filter expect to have at least one value in min or max numeric params');
}
this.min = filterParams.min;
this.max = filterParams.max;
this.columnType = filterParams.columnType;
}
module.exports = Range;
Range.prototype.sql = function(rawSql) {
var minMaxFilter;
if (Number.isFinite(this.min) && Number.isFinite(this.max)) {
minMaxFilter = betweenFilterTpl({
_column: this.column,
_min: this.min,
_max: this.max
});
} else if (Number.isFinite(this.min)) {
minMaxFilter = minFilterTpl({ _column: this.column, _min: this.min });
} else {
minMaxFilter = maxFilterTpl({ _column: this.column, _max: this.max });
}
return filterQueryTpl({
_sql: rawSql,
_filter: minMaxFilter
});
};

View File

@ -0,0 +1,25 @@
function SqlWrapMapConfigAdapter() {
}
module.exports = SqlWrapMapConfigAdapter;
SqlWrapMapConfigAdapter.prototype.getMapConfig = function(requestMapConfig, callback) {
if (requestMapConfig && Array.isArray(requestMapConfig.layers)) {
requestMapConfig.layers = requestMapConfig.layers.map(function(layer) {
if (layer.options) {
var sqlQueryWrap = layer.options.sql_wrap;
if (sqlQueryWrap) {
var layerSql = layer.options.sql;
if (layerSql) {
layer.options.sql_raw = layerSql;
layer.options.sql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, layerSql);
}
}
}
return layer;
});
}
return callback(null, requestMapConfig);
};

View File

@ -6,6 +6,7 @@ var step = require('step');
var MapConfig = require('windshaft').model.MapConfig;
var templateName = require('../../backends/template_maps').templateName;
var QueryTables = require('cartodb-query-tables');
var SqlWrapMapConfigAdapter = require('./adapter/sql-wrap-mapconfig-adapter');
/**
* @constructor
@ -22,6 +23,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend,
this.turboCartoAdapter = turboCartoAdapter;
this.analysisMapConfigAdapter = analysisMapConfigAdapter;
this.overviewsAdapter = overviewsAdapter;
this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter();
this.owner = owner;
this.templateName = templateName(templateId);
@ -95,6 +97,10 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
assert.ifError(err);
return self.templateMaps.instance(self.template, templateParams);
},
function prepareSqlWrap(err, requestMapConfig) {
assert.ifError(err);
self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this);
},
function prepareAnalysisLayers(err, requestMapConfig) {
assert.ifError(err);
var analysisConfiguration = {
@ -142,7 +148,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
assert.ifError(err);
var next = this;
self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, function(err, layers) {
self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, self.analysesResults, function(err, layers) {
if (err) {
return next(err);
}

View File

@ -1,13 +1,15 @@
var step = require('step');
var queue = require('queue-async');
var _ = require('underscore');
function MapConfigOverviewsAdapter(overviewsMetadataApi) {
function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) {
this.overviewsMetadataApi = overviewsMetadataApi;
this.filterStatsApi = filterStatsApi;
}
module.exports = MapConfigOverviewsAdapter;
MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) {
MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analysesResults, callback) {
var self = this;
if (!layers || layers.length === 0) {
@ -24,11 +26,52 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callb
if (err) {
done(err, layer);
} else {
if ( !_.isEmpty(metadata) ) {
layer = _.extend({}, layer);
layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } });
}
done(null, layer);
var query_rewrite_data = { overviews: metadata };
step(
function collectFiltersData() {
var filters, unfiltered_query;
if ( layer.options.source && analysesResults ) {
var sourceId = layer.options.source.id;
var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
if ( node ) {
node = node.rootNode;
filters = node.filters; // TODO: node.getFilters() when available in camshaft
var filters_disabler = Object.keys(filters).reduce(
function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
{}
);
unfiltered_query = node.getQuery(filters_disabler);
query_rewrite_data.filters = filters;
query_rewrite_data.unfiltered_query = unfiltered_query;
}
}
this(null, filters, unfiltered_query);
},
function collectStatsData(err, filters, unfiltered_query) {
var next_step = this;
if ( filters ) {
self.filterStatsApi.getFilterStats(
username,
unfiltered_query, filters,
function(err, stats) {
if ( !err ) {
query_rewrite_data.filter_stats = stats;
}
return next_step(err);
}
);
} else {
return next_step(null);
}
},
function addDataToLayer(err) {
if ( !err && !_.isEmpty(metadata) ) {
layer = _.extend({}, layer);
layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data });
}
done(err, layer);
}
);
}
});
}

View File

@ -21,6 +21,7 @@ var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
var FilterStatsApi = require('./api/filter_stats_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
@ -59,6 +60,7 @@ module.exports = function(serverOptions) {
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
@ -148,7 +150,7 @@ module.exports = function(serverOptions) {
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
var turboCartoParser = new TurboCartoParser(pgQueryRunner);
var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser);

View File

@ -1,5 +1,26 @@
var _ = require('underscore');
var TableNameParser = require('./table_name_parser');
var BBoxFilter = require('../models/filter/bbox');
var CamshaftFilter = require('../models/filter/camshaft');
// Minimim number of filtered rows to use overviews
var FILTER_MIN_ROWS = 65536;
// Maximum filtered fraction to not apply overviews
var FILTER_MAX_FRACTION = 0.2;
function apply_filters_to_query(query, filters, bbox_filter) {
if ( filters && !_.isEmpty(filters)) {
var camshaftFilter = new CamshaftFilter(filters);
query = camshaftFilter.sql(query);
}
if ( bbox_filter ) {
var bboxFilter = new BBoxFilter(bbox_filter.options, bbox_filter.params);
query = bboxFilter.sql(query);
}
return query;
}
function OverviewsQueryRewriter(options) {
this.options = options;
@ -119,26 +140,34 @@ function replace_table_in_query(sql, old_table_name, replacement) {
return sql.replace(new RegExp(regexp, 'g'), replacement);
}
function overviews_query(query, overviews, zoom_level_expression) {
function replace_table_in_query_with_schema(query, table, schema, replacement) {
if ( replacement ) {
query = replace_table_in_query(query, table, replacement);
var parsed_table = TableNameParser.parse(table);
if (!parsed_table.schema && schema) {
// replace also the qualified table name, if the table wasn't qualified
parsed_table.schema = schema;
table = TableNameParser.table_identifier(parsed_table);
query = replace_table_in_query(query, table, replacement);
}
}
return query;
}
// Build query to use overviews for a variant zoom level (given by a expression to
// be evaluated by the database server)
function overviews_query_with_zoom_expression(query, overviews, zoom_level_expression) {
var replaced_query = query;
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
var replacement;
for ( var table in overviews ) {
if (overviews.hasOwnProperty(table)) {
var table_overviews = overviews[table];
var table_view = overviews_view_name(table);
var schema = table_overviews.schema;
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
replaced_query = replace_table_in_query(replaced_query, table, replacement);
var parsed_table = TableNameParser.parse(table);
if (!parsed_table.schema && schema) {
// replace also the qualified table name, if the table wasn't qualified
parsed_table.schema = schema;
table = TableNameParser.table_identifier(parsed_table);
replaced_query = replace_table_in_query(replaced_query, table, replacement);
}
}
}
_.each(Object.keys(overviews), function(table) {
var table_overviews = overviews[table];
var table_view = overviews_view_name(table);
var schema = table_overviews.schema;
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement);
});
if ( replaced_query !== query ) {
sql += "\n";
sql += replaced_query;
@ -148,34 +177,128 @@ function overviews_query(query, overviews, zoom_level_expression) {
return sql;
}
// Build query to use overviews for a specific zoom level value
function overviews_query_with_definite_zoom(query, overviews, zoom_level) {
var replaced_query = query;
var replacement;
_.each(Object.keys(overviews), function(table) {
var table_overviews = overviews[table];
var schema = table_overviews.schema;
replacement = overview_table_for_zoom_level(table_overviews, zoom_level);
replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement);
});
return replaced_query;
}
// Find a suitable overview table for a specific zoom_level
function overview_table_for_zoom_level(table_overviews, zoom_level) {
var overview_table;
if ( table_overviews ) {
overview_table = table_overviews[zoom_level];
if ( !overview_table ) {
_.every(Object.keys(table_overviews).sort(function(x,y){ return x-y; }), function(overview_zoom) {
if ( +overview_zoom > +zoom_level ) {
overview_table = table_overviews[overview_zoom];
return false;
} else {
return true;
}
});
}
}
if ( overview_table ) {
overview_table = overview_table.table;
}
return overview_table;
}
// Transform an SQL query so that it uses overviews.
// overviews contains metadata about the overviews to be used:
// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
//
// For a given query `SELECT * FROM table`, if any of tables in it
// has overviews as defined by the provided metadat, the query will
// be transform into something similar to this:
//
// WITH _vovw_scale AS ( ... ), -- define scale level
// WITH _vovw_table AS ( ... ), -- define union of overviews and base table
// SELECT * FROM _vovw_table -- query with table replaced by _vovw_table
// SELECT * FROM -- in the query the table is replaced by:
// ( ... ) AS _vovw_table -- a union of overviews and base table
//
// This transformation can in principle be applied to arbitrary queries
// (except for the case of queries that include the name of tables with
// overviews inside text literals: at the current table name substitution
// doesnn't prevent substitution inside literals).
// But the transformation will currently only be applied to simple queries
// of the form detected by the overviews_supported_query function.
OverviewsQueryRewriter.prototype.query = function(query, data) {
var overviews = this.overviews_metadata(data);
if ( !overviews || !this.is_supported_query(query)) {
// The data argument has the form:
// {
// overviews: // overview tables metadata
// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... },
// zoom_level: ..., // optional zoom level
// filters: ..., // filters definition
// unfiltered_query: ..., // query without the filters
// bbox_filter: ... // bounding-box filter
// }
OverviewsQueryRewriter.prototype.query = function(query, data, options) {
options = options || {};
data = data || {};
var overviews = data.overviews;
var unfiltered_query = data.unfiltered_query;
var filters = data.filters;
var bbox_filter = data.bbox_filter;
if ( !unfiltered_query ) {
unfiltered_query = query;
}
if ( !should_use_overviews(unfiltered_query, data) ) {
return query;
}
var zoom_level_expression = this.options.zoom_level || '0';
return overviews_query(query, overviews, zoom_level_expression);
var rewritten_query;
var zoom_level_expression = this.options.zoom_level;
var zoom_level = zoom_level_for_query(unfiltered_query, zoom_level_expression, options);
rewritten_query = overviews_query(unfiltered_query, overviews, zoom_level, zoom_level_expression);
if ( rewritten_query === unfiltered_query ) {
// could not or didn't need to alter the query
rewritten_query = query;
} else {
rewritten_query = apply_filters_to_query(rewritten_query, filters, bbox_filter);
}
return rewritten_query;
};
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
function zoom_level_for_query(query, zoom_level_expression, options) {
var zoom_level = null;
if ( _.has(options, 'zoom_level') ) {
zoom_level = options.zoom_level || '0';
}
if ( zoom_level === null && !zoom_level_expression ) {
zoom_level = '0';
}
return zoom_level;
}
function overviews_query(query, overviews, zoom_level, zoom_level_expression) {
if ( zoom_level || zoom_level === '0' || zoom_level === 0 ) {
return overviews_query_with_definite_zoom(query, overviews, zoom_level);
} else {
return overviews_query_with_zoom_expression(query, overviews, zoom_level_expression);
}
}
function should_use_overviews(query, data) {
data = data || {};
var use_overviews = data.overviews && is_supported_query(query);
if ( use_overviews && data.filters && data.filter_stats ) {
var filtered_rows = data.filter_stats.filtered_rows;
var unfiltered_rows = data.filter_stats.unfiltered_rows;
if ( unfiltered_rows && (filtered_rows || filtered_rows === 0) ) {
use_overviews = filtered_rows >= FILTER_MIN_ROWS ||
(filtered_rows/unfiltered_rows) > FILTER_MAX_FRACTION;
}
}
return use_overviews;
}
function is_supported_query(sql) {
var basic_query =
/\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i');
@ -187,8 +310,4 @@ OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
'i'
);
return !!(sql.match(unwrapped_query) || sql.match(wrapped_query));
};
OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {
return data && data.overviews;
};
}

2
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "2.42.3",
"version": "2.44.0",
"dependencies": {
"body-parser": {
"version": "1.14.2",

View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "2.42.3",
"version": "2.44.0",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"

View File

@ -0,0 +1,358 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('dataviews using tables without overviews', function() {
var nonOverviewsMapConfig = {
version: '1.5.0',
analyses: [
{ id: 'data-source',
type: 'source',
params: {
query: 'select * from populated_places_simple_reduced'
}
}
],
dataviews: {
country_places_count: {
type: 'formula',
source: {id: 'data-source'},
options: {
column: 'adm0_a3',
operation: 'count'
}
},
country_categories: {
type: 'aggregation',
source: {id: 'data-source'},
options: {
column: 'adm0_a3',
aggregation: 'count'
}
}
},
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
source: { id: 'data-source' }
}
}
]
};
it("should expose a formula", function(done) {
var testClient = new TestClient(nonOverviewsMapConfig);
testClient.getDataview('country_places_count', { own_filter: 0 }, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, { operation: 'count', result: 7313, nulls: 0, type: 'formula' });
testClient.drain(done);
});
});
it("should admit a bbox", function(done) {
var params = {
bbox: "-170,-80,170,80"
};
var testClient = new TestClient(nonOverviewsMapConfig);
testClient.getDataview('country_places_count', params, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, { operation: 'count', result: 7253, nulls: 0, type: 'formula' });
testClient.drain(done);
});
});
describe('filters', function() {
describe('category', function () {
it("should expose a filtered formula", function (done) {
var params = {
filters: {
dataviews: {country_categories: {accept: ['CAN']}}
}
};
var testClient = new TestClient(nonOverviewsMapConfig);
testClient.getDataview('country_places_count', params, function (err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, { operation: 'count', result: 256, nulls: 0, type: 'formula' });
testClient.drain(done);
});
});
it("should expose a filtered formula and admit a bbox", function (done) {
var params = {
filters: {
dataviews: {country_categories: {accept: ['CAN']}}
},
bbox: "-170,-80,170,80"
};
var testClient = new TestClient(nonOverviewsMapConfig);
testClient.getDataview('country_places_count', params, function (err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, { operation: 'count', result: 254, nulls: 0, type: 'formula' });
testClient.drain(done);
});
});
});
});
});
describe('dataviews using tables with overviews', function() {
var overviewsMapConfig = {
version: '1.5.0',
analyses: [
{ id: 'data-source',
type: 'source',
params: {
query: 'select * from test_table_overviews'
}
}
],
dataviews: {
test_sum: {
type: 'formula',
source: {id: 'data-source'},
options: {
column: 'value',
operation: 'sum'
}
},
test_categories: {
type: 'aggregation',
source: {id: 'data-source'},
options: {
column: 'name',
aggregation: 'count',
aggregationColumn: 'name',
}
},
test_avg: {
type: 'formula',
source: {id: 'data-source'},
options: {
column: 'value',
operation: 'avg'
}
},
test_count: {
type: 'formula',
source: {id: 'data-source'},
options: {
column: 'value',
operation: 'count'
}
},
test_min: {
type: 'formula',
source: {id: 'data-source'},
options: {
column: 'value',
operation: 'min'
}
},
test_max: {
type: 'formula',
source: {id: 'data-source'},
options: {
column: 'value',
operation: 'max'
}
}
},
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table_overviews',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
source: { id: 'data-source' }
}
}
]
};
it("should expose a sum formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_sum', { own_filter: 0 }, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose an avg formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_avg', { own_filter: 0 }, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a count formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_count', { own_filter: 0 }, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a max formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_max', { own_filter: 0 }, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a min formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_min', { own_filter: 0 }, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should admit a bbox", function(done) {
var params = {
bbox: "-170,-80,170,80"
};
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_sum', params, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
describe('filters', function() {
describe('category', function () {
var params = {
filters: {
dataviews: {test_categories: {accept: ['Hawai']}}
}
};
it("should expose a filtered sum formula", function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_sum', params, function (err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a filtered avg formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_avg', params, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a filtered count formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_count', params, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a filterd max formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_max', params, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a filterd min formula", function(done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_min', params, function(err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
it("should expose a filtered sum formula with bbox", function (done) {
var bboxparams = {
filters: {
dataviews: {test_categories: {accept: ['Hawai']}}
},
bbox: "-170,-80,170,80"
};
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_sum', bboxparams, function (err, formula_result) {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
testClient.drain(done);
});
});
});
});
});

View File

@ -109,3 +109,118 @@ describe('overviews metadata', function() {
);
});
});
describe('overviews metadata with filters', function() {
// configure redis pool instance to use in tests
var redisPool = new RedisPool(global.environment.redis);
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
test_helper.deleteRedisKeys(keysToDelete, done);
});
it("layers with overviews", function(done) {
var layergroup = {
version: '1.5.0',
layers: [
{
type: 'cartodb',
options: {
sql: 'SELECT * FROM test_table_overviews',
source: { id: 'with_overviews' },
cartocss: '#layer { marker-fill: black; }',
cartocss_version: '2.3.0'
}
}
],
dataviews: {
test_names: {
type: 'aggregation',
source: {id: 'with_overviews'},
options: {
column: 'name',
aggregation: 'count'
}
}
},
analyses: [
{ id: 'with_overviews',
type: 'source',
params: {
query: 'select * from test_table_overviews'
}
}
]
};
var filters = {
dataviews: {
test_names: { accept: ['Hawai'] }
}
};
var layergroup_url = '/api/v1/map';
var expected_token;
step(
function do_post()
{
var next = this;
assert.response(server, {
url: layergroup_url + '?filters=' + JSON.stringify(filters),
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
expected_token = parsedBody.layergroupid;
next(null, res);
});
},
function do_get_mapconfig(err)
{
assert.ifError(err);
var next = this;
var mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: 500000
});
mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) {
assert.ifError(err);
assert.equal(mapConfig._cfg.layers[0].type, 'cartodb');
assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data);
var expected_data = {
overviews: {
test_table_overviews: {
schema: 'public',
1: { table: '_vovw_1_test_table_overviews' },
2: { table: '_vovw_2_test_table_overviews' }
}
},
filters: { test_names: { type: 'category', column: 'name', params: { accept: [ 'Hawai' ] } } },
unfiltered_query: 'select * from test_table_overviews',
filter_stats: { unfiltered_rows: 5, filtered_rows: 1 }
};
assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data);
});
next(err);
},
function finish(err) {
keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done(err);
}
);
});
});

View File

@ -0,0 +1,52 @@
require('../support/test_helper');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
describe('sql-wrap', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
return done();
}
});
it('should use sql_wrap from layer options', function(done) {
var mapConfig = {
version: '1.5.0',
layers: [
{
"type": "cartodb",
"options": {
"sql": "SELECT * FROM populated_places_simple_reduced",
"sql_wrap": "SELECT * FROM (<%= sql %>) _w WHERE adm0_a3 = 'USA'",
"cartocss": [
"#points {",
" marker-fill-opacity: 1;",
" marker-line-color: #FFF;",
" marker-line-width: 0.5;",
" marker-line-opacity: 1;",
" marker-placement: point;",
" marker-type: ellipse;",
" marker-width: 8;",
" marker-fill: red;",
" marker-allow-overlap: true;",
"}"
].join('\n'),
"cartocss_version": "2.3.0"
}
}
]
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getTile(0, 0, 0, function(err, tile, img) {
assert.ok(!err, err);
var fixtureImg = './test/fixtures/sql-wrap-usa-filter.png';
assert.imageIsSimilarToFile(img, fixtureImg, 20, done);
});
});
});

BIN
test/fixtures/sql-wrap-usa-filter.png vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.1 KiB

View File

@ -6,6 +6,7 @@ var cartodbRedis = require('cartodb-redis');
var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api');
var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter');
// configure redis pool instance to use in tests
@ -17,9 +18,9 @@ var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
describe('MapConfigOverviewsAdapter', function() {
@ -36,7 +37,7 @@ describe('MapConfigOverviewsAdapter', function() {
}
};
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
assert.ok(!err);
assert.equal(layers.length, 1);
assert.equal(layers[0].type, 'cartodb');
@ -64,7 +65,7 @@ describe('MapConfigOverviewsAdapter', function() {
}
};
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
assert.ok(!err);
assert.equal(layers.length, 1);
assert.equal(layers[0].type, 'cartodb');

View File

@ -247,8 +247,10 @@ CREATE TABLE test_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
@ -274,11 +276,11 @@ SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true);
ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass);
INSERT INTO test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 3.0, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 5.0, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1);
ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id);
@ -294,8 +296,10 @@ CREATE TABLE _vovw_1_test_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
@ -313,8 +317,10 @@ CREATE TABLE _vovw_2_test_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
@ -327,11 +333,11 @@ GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER;
INSERT INTO _vovw_2_test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 8.0/3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 3),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 7.0/2.0, '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 2);
INSERT INTO _vovw_1_test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241');
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5);
-- analysis tables -----------------------------------------------

View File

@ -475,4 +475,137 @@ describe('Overviews query rewriter', function() {
";
assertSameSql(overviews_sql, expected_sql);
});
it('generates query for specific Z level', function(){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
2: { table: 'table1_ov2' },
3: { table: 'table1_ov3' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 3 });
var expected_sql = "SELECT * FROM table1_ov3";
assertSameSql(overviews_sql, expected_sql);
});
it('generates query for specific nonpresent Z level', function(){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
2: { table: 'table1_ov2' },
3: { table: 'table1_ov3' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 1 });
var expected_sql = "SELECT * FROM table1_ov2";
assertSameSql(overviews_sql, expected_sql);
});
it('does not use overviews for specific out-of-range Z level', function(){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
2: { table: 'table1_ov2' },
3: { table: 'table1_ov3' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 4 });
var expected_sql = "SELECT * FROM table1";
assertSameSql(overviews_sql, expected_sql);
});
it('generates query with filters', function(){
var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
FROM (SELECT *\
FROM (select * from table1) _camshaft_category_filter\
WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
1: { table: 'table1_ov1' },
2: { table: 'table1_ov2' },
3: { table: 'table1_ov3' }
}
},
filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
unfiltered_query: 'SELECT * FROM table1'
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
SELECT * FROM (WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
UNION ALL\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\
UNION ALL\
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
) AS _vovw_table1) _camshaft_category_filter\
WHERE name IN ($escape_0$X$escape_0$)\
";
assertSameSql(overviews_sql, expected_sql);
});
it('generates query with filters for specific zoom level', function(){
var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
FROM (SELECT *\
FROM (select * from table1) _camshaft_category_filter\
WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
1: { table: 'table1_ov1' },
2: { table: 'table1_ov2' },
3: { table: 'table1_ov3' }
}
},
filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
unfiltered_query: 'SELECT * FROM table1',
filter_stats: { unfiltered_rows: 1000, filtered_rows: 900 }
};
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 2 });
var expected_sql = "\
SELECT * FROM (SELECT * FROM table1_ov2) _camshaft_category_filter\
WHERE name IN ($escape_0$X$escape_0$)\
";
assertSameSql(overviews_sql, expected_sql);
});
it('does not generates query with aggressive filtering', function(){
var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
FROM (SELECT *\
FROM (select * from table1) _camshaft_category_filter\
WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
1: { table: 'table1_ov1' },
2: { table: 'table1_ov2' },
3: { table: 'table1_ov3' }
}
},
filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
unfiltered_query: 'SELECT * FROM table1',
filter_stats: { unfiltered_rows: 1000, filtered_rows: 10 }
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
});
});