Merge branch 'master' into turbo-carto-category
This commit is contained in:
commit
3bacfecc49
20
NEWS.md
20
NEWS.md
@ -1,9 +1,27 @@
|
||||
# Changelog
|
||||
|
||||
## 2.42.3
|
||||
## 2.44.0
|
||||
|
||||
Released 2016-mm-dd
|
||||
|
||||
Adds support for sql wrap in all layers
|
||||
|
||||
|
||||
## 2.43.1
|
||||
|
||||
Released 2016-05-19
|
||||
|
||||
Bug fixes:
|
||||
- Dataview error when bbox present without query rewrite data #458
|
||||
|
||||
|
||||
## 2.43.0
|
||||
|
||||
Released 2016-05-18
|
||||
|
||||
New features:
|
||||
- Overviews now support dataviews and filtering #449
|
||||
|
||||
|
||||
## 2.42.2
|
||||
|
||||
|
59
lib/cartodb/api/filter_stats_api.js
Normal file
59
lib/cartodb/api/filter_stats_api.js
Normal file
@ -0,0 +1,59 @@
|
||||
var _ = require('underscore');
|
||||
var step = require('step');
|
||||
var CamshaftFilter = require('../models/filter/camshaft');
|
||||
|
||||
function FilterStatsApi(pgQueryRunner) {
|
||||
this.pgQueryRunner = pgQueryRunner;
|
||||
}
|
||||
|
||||
module.exports = FilterStatsApi;
|
||||
|
||||
function getEstimatedRows(pgQueryRunner, username, query, callback) {
|
||||
pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) {
|
||||
if (err){
|
||||
callback(err);
|
||||
return;
|
||||
}
|
||||
var rows;
|
||||
if ( result_rows[0] && result_rows[0]['QUERY PLAN'] &&
|
||||
result_rows[0]['QUERY PLAN'][0] && result_rows[0]['QUERY PLAN'][0].Plan ) {
|
||||
rows = result_rows[0]['QUERY PLAN'][0].Plan['Plan Rows'];
|
||||
}
|
||||
return callback(null, rows);
|
||||
});
|
||||
}
|
||||
|
||||
FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) {
|
||||
var stats = {};
|
||||
var self = this;
|
||||
step(
|
||||
function getUnfilteredRows() {
|
||||
getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this);
|
||||
},
|
||||
function receiveUnfilteredRows(err, rows) {
|
||||
if (err){
|
||||
callback(err);
|
||||
return;
|
||||
}
|
||||
stats.unfiltered_rows = rows;
|
||||
this(null, rows);
|
||||
},
|
||||
function getFilteredRows() {
|
||||
if ( filters && !_.isEmpty(filters)) {
|
||||
var camshaftFilter = new CamshaftFilter(filters);
|
||||
var query = camshaftFilter.sql(unfiltered_query);
|
||||
getEstimatedRows(self.pgQueryRunner, username, query, this);
|
||||
} else {
|
||||
this(null, null);
|
||||
}
|
||||
},
|
||||
function receiveFilteredRows(err, rows) {
|
||||
if (err){
|
||||
callback(err);
|
||||
return;
|
||||
}
|
||||
stats.filtered_rows = rows;
|
||||
callback(null, stats);
|
||||
}
|
||||
);
|
||||
};
|
@ -8,7 +8,13 @@ var step = require('step');
|
||||
var Timer = require('../stats/timer');
|
||||
|
||||
var BBoxFilter = require('../models/filter/bbox');
|
||||
|
||||
var DataviewFactory = require('../models/dataview/factory');
|
||||
var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
|
||||
var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter');
|
||||
var overviewsQueryRewriter = new OverviewsQueryRewriter({
|
||||
zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
|
||||
});
|
||||
|
||||
function DataviewBackend(analysisBackend) {
|
||||
this.analysisBackend = analysisBackend;
|
||||
@ -16,7 +22,6 @@ function DataviewBackend(analysisBackend) {
|
||||
|
||||
module.exports = DataviewBackend;
|
||||
|
||||
|
||||
DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
|
||||
var self = this;
|
||||
|
||||
@ -101,6 +106,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
|
||||
ownFilter = !!ownFilter;
|
||||
|
||||
var query;
|
||||
|
||||
if (ownFilter) {
|
||||
query = node.getQuery();
|
||||
} else {
|
||||
@ -109,11 +115,48 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
|
||||
query = node.getQuery(applyFilters);
|
||||
}
|
||||
|
||||
var sourceId = dataviewDefinition.source.id; // node.id
|
||||
var layer = _.find(
|
||||
mapConfig.obj().layers,
|
||||
function(l){ return l.options.source && (l.options.source.id === sourceId); }
|
||||
);
|
||||
var queryRewriteData = layer && layer.options.query_rewrite_data;
|
||||
if ( queryRewriteData ) {
|
||||
if ( node.type === 'source' ) {
|
||||
var filters = node.filters; // TODO: node.getFilters() when available in camshaft
|
||||
var filters_disabler = Object.keys(filters).reduce(
|
||||
function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
|
||||
{}
|
||||
);
|
||||
var unfiltered_query = node.getQuery(filters_disabler);
|
||||
queryRewriteData = _.extend(
|
||||
{},
|
||||
queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (params.bbox) {
|
||||
var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
|
||||
query = bboxFilter.sql(query);
|
||||
if ( queryRewriteData ) {
|
||||
var bbox_filter_definition = {
|
||||
type: 'bbox',
|
||||
options: {
|
||||
column: 'the_geom',
|
||||
srid: 4326,
|
||||
},
|
||||
params: {
|
||||
bbox: params.bbox
|
||||
}
|
||||
};
|
||||
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
|
||||
}
|
||||
}
|
||||
|
||||
var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
|
||||
overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
|
||||
);
|
||||
|
||||
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
|
||||
function castNumbers(overrides, val, k) {
|
||||
@ -123,7 +166,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
|
||||
{ownFilter: ownFilter}
|
||||
);
|
||||
|
||||
var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
|
||||
var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
|
||||
dataview.getResult(pg, overrideParams, this);
|
||||
},
|
||||
function returnCallback(err, result) {
|
||||
@ -277,4 +320,4 @@ function dbParamsFromReqParams(params) {
|
||||
dbParams.dbname = params.dbname;
|
||||
}
|
||||
return dbParams;
|
||||
}
|
||||
}
|
||||
|
@ -19,6 +19,7 @@ var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adap
|
||||
var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter');
|
||||
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
|
||||
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
|
||||
var SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
|
||||
|
||||
/**
|
||||
* @param {AuthApi} authApi
|
||||
@ -52,6 +53,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
|
||||
this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend);
|
||||
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
|
||||
this.overviewsAdapter = overviewsAdapter;
|
||||
this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter();
|
||||
}
|
||||
|
||||
util.inherits(MapController, BaseController);
|
||||
@ -139,6 +141,10 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
|
||||
self.req2params(req, this);
|
||||
},
|
||||
prepareConfigFn,
|
||||
function prepareSqlWrap(err, requestMapConfig) {
|
||||
assert.ifError(err);
|
||||
self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this);
|
||||
},
|
||||
function prepareAnalysisLayers(err, requestMapConfig) {
|
||||
assert.ifError(err);
|
||||
var analysisConfiguration = {
|
||||
@ -185,17 +191,20 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
|
||||
function addOverviewsInformation(err, requestMapConfig, datasource) {
|
||||
assert.ifError(err);
|
||||
var next = this;
|
||||
self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
self.overviewsAdapter.getLayers(
|
||||
req.context.user, requestMapConfig.layers, analysesResults,
|
||||
function(err, layers) {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
|
||||
if (layers) {
|
||||
requestMapConfig.layers = layers;
|
||||
}
|
||||
if (layers) {
|
||||
requestMapConfig.layers = layers;
|
||||
}
|
||||
|
||||
return next(null, requestMapConfig, datasource);
|
||||
});
|
||||
return next(null, requestMapConfig, datasource);
|
||||
}
|
||||
);
|
||||
},
|
||||
function parseTurboCarto(err, requestMapConfig, datasource) {
|
||||
assert.ifError(err);
|
||||
|
32
lib/cartodb/models/dataview/overviews/factory.js
Normal file
32
lib/cartodb/models/dataview/overviews/factory.js
Normal file
@ -0,0 +1,32 @@
|
||||
var parentFactory = require('../factory');
|
||||
var dataviews = require('./');
|
||||
|
||||
function OverviewsDataviewFactory(queryRewriter, queryRewriteData, options) {
|
||||
this.queryRewriter = queryRewriter;
|
||||
this.queryRewriteData = queryRewriteData;
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinition) {
|
||||
var type = dataviewDefinition.type;
|
||||
var dataviews = OverviewsDataviewMetaFactory.dataviews;
|
||||
if ( !this.queryRewriter || !this.queryRewriteData || !dataviews[type] ) {
|
||||
return parentFactory.getDataview(query, dataviewDefinition);
|
||||
}
|
||||
return new dataviews[type](
|
||||
query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
|
||||
);
|
||||
};
|
||||
|
||||
var OverviewsDataviewMetaFactory = {
|
||||
dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
|
||||
allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
|
||||
return allDataviews;
|
||||
}, {}),
|
||||
|
||||
getFactory: function(queryRewriter, queryRewriteData, options) {
|
||||
return new OverviewsDataviewFactory(queryRewriter, queryRewriteData, options);
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = OverviewsDataviewMetaFactory;
|
104
lib/cartodb/models/dataview/overviews/formula.js
Normal file
104
lib/cartodb/models/dataview/overviews/formula.js
Normal file
@ -0,0 +1,104 @@
|
||||
var _ = require('underscore');
|
||||
var BaseWidget = require('../base');
|
||||
var BaseDataview = require('../formula');
|
||||
|
||||
var debug = require('debug')('windshaft:widget:formula:overviews');
|
||||
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var formulaQueryTpls = {
|
||||
'count': dot.template([
|
||||
'SELECT',
|
||||
'sum(_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
].join('\n')),
|
||||
'sum': dot.template([
|
||||
'SELECT',
|
||||
'sum({{=it._column}}*_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
].join('\n')),
|
||||
'avg': dot.template([
|
||||
'SELECT',
|
||||
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
].join('\n')),
|
||||
};
|
||||
|
||||
function Formula(query, options, queryRewriter, queryRewriteData, params) {
|
||||
this.base_dataview = new BaseDataview(query, options);
|
||||
this.query = query;
|
||||
this.column = options.column || '1';
|
||||
this.operation = options.operation;
|
||||
this.queryRewriter = queryRewriter;
|
||||
this.queryRewriteData = queryRewriteData;
|
||||
this.options = params;
|
||||
}
|
||||
|
||||
Formula.prototype = new BaseWidget();
|
||||
Formula.prototype.constructor = Formula;
|
||||
|
||||
module.exports = Formula;
|
||||
|
||||
var zoom_level_factor = 100.0;
|
||||
|
||||
// Compute zoom level so that the the resolution grid size of the
|
||||
// selected overview is smaller (zoom_level_factor times smaller at least)
|
||||
// than the bounding box size.
|
||||
function zoom_level_for_bbox(bbox) {
|
||||
var px_per_tile = 256.0;
|
||||
var earth_width = 360.0;
|
||||
// TODO: now we assume overviews are computed for 1-pixel tolerance;
|
||||
// should use extended overviews metadata to compute this properly.
|
||||
if ( bbox ) {
|
||||
var bbox_values = _.map(bbox.split(','), function(v) { return +v; });
|
||||
var w = Math.abs(bbox_values[2]-bbox_values[0]);
|
||||
var h = Math.abs(bbox_values[3]-bbox_values[1]);
|
||||
var max_dim = Math.min(w, h);
|
||||
|
||||
// Find minimum suitable z
|
||||
// note that the QueryRewirter will use the minimum level overview
|
||||
// of level >= z if it exists, and otherwise the base table
|
||||
var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0));
|
||||
return Math.max(z, 0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
Formula.prototype.sql = function(psql, filters, override, callback) {
|
||||
var _query = this.query;
|
||||
var formulaQueryTpl = formulaQueryTpls[this.operation];
|
||||
|
||||
if ( formulaQueryTpl ) {
|
||||
// supported formula for use with overviews
|
||||
var zoom_level = zoom_level_for_bbox(this.options.bbox);
|
||||
_query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level });
|
||||
var formulaSql = formulaQueryTpl({
|
||||
_query: _query,
|
||||
_operation: this.operation,
|
||||
_column: this.column
|
||||
});
|
||||
debug(formulaSql);
|
||||
callback = callback || override;
|
||||
|
||||
return callback(null, formulaSql);
|
||||
}
|
||||
|
||||
// For non supported operations (min, max) we're not using overviews.
|
||||
return this.base_dataview.sql(psql, filters, override, callback);
|
||||
};
|
||||
|
||||
Formula.prototype.format = function(result) {
|
||||
return this.base_dataview.format(result);
|
||||
};
|
||||
|
||||
Formula.prototype.getType = function() {
|
||||
return this.base_dataview.getType();
|
||||
};
|
||||
|
||||
Formula.prototype.toString = function() {
|
||||
return this.base_dataview.toString();
|
||||
};
|
3
lib/cartodb/models/dataview/overviews/index.js
Normal file
3
lib/cartodb/models/dataview/overviews/index.js
Normal file
@ -0,0 +1,3 @@
|
||||
module.exports = {
|
||||
Formula: require('./formula')
|
||||
};
|
35
lib/cartodb/models/filter/camshaft.js
Normal file
35
lib/cartodb/models/filter/camshaft.js
Normal file
@ -0,0 +1,35 @@
|
||||
var filters = {
|
||||
category: require('./camshaft/category'),
|
||||
range: require('./camshaft/range')
|
||||
};
|
||||
|
||||
function createFilter(filterDefinition) {
|
||||
var filterType = filterDefinition.type.toLowerCase();
|
||||
if (!filters.hasOwnProperty(filterType)) {
|
||||
throw new Error('Unknown filter type: ' + filterType);
|
||||
}
|
||||
return new filters[filterType](filterDefinition.column, filterDefinition.params);
|
||||
}
|
||||
|
||||
function CamshaftFilters(filters) {
|
||||
this.filters = filters;
|
||||
}
|
||||
|
||||
CamshaftFilters.prototype.sql = function(rawSql) {
|
||||
var filters = this.filters || {};
|
||||
var applyFilters = {};
|
||||
|
||||
return Object.keys(filters)
|
||||
.filter(function(filterName) {
|
||||
return applyFilters.hasOwnProperty(filterName) ? applyFilters[filterName] : true;
|
||||
})
|
||||
.map(function(filterName) {
|
||||
var filterDefinition = filters[filterName];
|
||||
return createFilter(filterDefinition);
|
||||
})
|
||||
.reduce(function(sql, filter) {
|
||||
return filter.sql(sql);
|
||||
}, rawSql);
|
||||
};
|
||||
|
||||
module.exports = CamshaftFilters;
|
79
lib/cartodb/models/filter/camshaft/category.js
Normal file
79
lib/cartodb/models/filter/camshaft/category.js
Normal file
@ -0,0 +1,79 @@
|
||||
'use strict';
|
||||
|
||||
var debug = require('debug')('windshaft:filter:category');
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var filterQueryTpl = dot.template([
|
||||
'SELECT *',
|
||||
'FROM ({{=it._sql}}) _camshaft_category_filter',
|
||||
'WHERE {{=it._filters}}'
|
||||
].join('\n'));
|
||||
var escapeStringTpl = dot.template('$escape_{{=it._i}}${{=it._value}}$escape_{{=it._i}}$');
|
||||
var inConditionTpl = dot.template('{{=it._column}} IN ({{=it._values}})');
|
||||
var notInConditionTpl = dot.template('{{=it._column}} NOT IN ({{=it._values}})');
|
||||
|
||||
function Category(column, filterParams) {
|
||||
this.column = column;
|
||||
|
||||
if (!Array.isArray(filterParams.accept) && !Array.isArray(filterParams.reject)) {
|
||||
throw new Error('Category filter expects at least one array in accept or reject params');
|
||||
}
|
||||
|
||||
if (Array.isArray(filterParams.accept) && Array.isArray(filterParams.reject)) {
|
||||
if (filterParams.accept.length === 0 && filterParams.reject.length === 0) {
|
||||
throw new Error(
|
||||
'Category filter expects one value either in accept or reject params when both are provided'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.accept = filterParams.accept;
|
||||
this.reject = filterParams.reject;
|
||||
}
|
||||
|
||||
module.exports = Category;
|
||||
|
||||
/*
|
||||
- accept: [] => reject all
|
||||
- reject: [] => accept all
|
||||
*/
|
||||
Category.prototype.sql = function(rawSql) {
|
||||
var valueFilters = [];
|
||||
|
||||
if (Array.isArray(this.accept)) {
|
||||
if (this.accept.length > 0) {
|
||||
valueFilters.push(inConditionTpl({
|
||||
_column: this.column,
|
||||
_values: this.accept.map(function(value, i) {
|
||||
return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value});
|
||||
}).join(',')
|
||||
}));
|
||||
} else {
|
||||
valueFilters.push('0 = 1');
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(this.reject)) {
|
||||
if (this.reject.length > 0) {
|
||||
valueFilters.push(notInConditionTpl({
|
||||
_column: this.column,
|
||||
_values: this.reject.map(function (value, i) {
|
||||
return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value});
|
||||
}).join(',')
|
||||
}));
|
||||
} else {
|
||||
valueFilters.push('1 = 1');
|
||||
}
|
||||
}
|
||||
|
||||
debug(filterQueryTpl({
|
||||
_sql: rawSql,
|
||||
_filters: valueFilters.join(' AND ')
|
||||
}));
|
||||
|
||||
return filterQueryTpl({
|
||||
_sql: rawSql,
|
||||
_filters: valueFilters.join(' AND ')
|
||||
});
|
||||
};
|
43
lib/cartodb/models/filter/camshaft/range.js
Normal file
43
lib/cartodb/models/filter/camshaft/range.js
Normal file
@ -0,0 +1,43 @@
|
||||
'use strict';
|
||||
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var betweenFilterTpl = dot.template('{{=it._column}} BETWEEN {{=it._min}} AND {{=it._max}}');
|
||||
var minFilterTpl = dot.template('{{=it._column}} >= {{=it._min}}');
|
||||
var maxFilterTpl = dot.template('{{=it._column}} <= {{=it._max}}');
|
||||
var filterQueryTpl = dot.template('SELECT * FROM ({{=it._sql}}) _camshaft_range_filter WHERE {{=it._filter}}');
|
||||
|
||||
function Range(column, filterParams) {
|
||||
this.column = column;
|
||||
|
||||
if (!Number.isFinite(filterParams.min) && !Number.isFinite(filterParams.max)) {
|
||||
throw new Error('Range filter expect to have at least one value in min or max numeric params');
|
||||
}
|
||||
|
||||
this.min = filterParams.min;
|
||||
this.max = filterParams.max;
|
||||
this.columnType = filterParams.columnType;
|
||||
}
|
||||
|
||||
module.exports = Range;
|
||||
|
||||
Range.prototype.sql = function(rawSql) {
|
||||
var minMaxFilter;
|
||||
if (Number.isFinite(this.min) && Number.isFinite(this.max)) {
|
||||
minMaxFilter = betweenFilterTpl({
|
||||
_column: this.column,
|
||||
_min: this.min,
|
||||
_max: this.max
|
||||
});
|
||||
} else if (Number.isFinite(this.min)) {
|
||||
minMaxFilter = minFilterTpl({ _column: this.column, _min: this.min });
|
||||
} else {
|
||||
minMaxFilter = maxFilterTpl({ _column: this.column, _max: this.max });
|
||||
}
|
||||
|
||||
return filterQueryTpl({
|
||||
_sql: rawSql,
|
||||
_filter: minMaxFilter
|
||||
});
|
||||
};
|
@ -0,0 +1,25 @@
|
||||
function SqlWrapMapConfigAdapter() {
|
||||
}
|
||||
|
||||
module.exports = SqlWrapMapConfigAdapter;
|
||||
|
||||
|
||||
SqlWrapMapConfigAdapter.prototype.getMapConfig = function(requestMapConfig, callback) {
|
||||
if (requestMapConfig && Array.isArray(requestMapConfig.layers)) {
|
||||
requestMapConfig.layers = requestMapConfig.layers.map(function(layer) {
|
||||
if (layer.options) {
|
||||
var sqlQueryWrap = layer.options.sql_wrap;
|
||||
if (sqlQueryWrap) {
|
||||
var layerSql = layer.options.sql;
|
||||
if (layerSql) {
|
||||
layer.options.sql_raw = layerSql;
|
||||
layer.options.sql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, layerSql);
|
||||
}
|
||||
}
|
||||
}
|
||||
return layer;
|
||||
});
|
||||
}
|
||||
|
||||
return callback(null, requestMapConfig);
|
||||
};
|
@ -6,6 +6,7 @@ var step = require('step');
|
||||
var MapConfig = require('windshaft').model.MapConfig;
|
||||
var templateName = require('../../backends/template_maps').templateName;
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
var SqlWrapMapConfigAdapter = require('./adapter/sql-wrap-mapconfig-adapter');
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
@ -22,6 +23,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend,
|
||||
this.turboCartoAdapter = turboCartoAdapter;
|
||||
this.analysisMapConfigAdapter = analysisMapConfigAdapter;
|
||||
this.overviewsAdapter = overviewsAdapter;
|
||||
this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter();
|
||||
|
||||
this.owner = owner;
|
||||
this.templateName = templateName(templateId);
|
||||
@ -95,6 +97,10 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
|
||||
assert.ifError(err);
|
||||
return self.templateMaps.instance(self.template, templateParams);
|
||||
},
|
||||
function prepareSqlWrap(err, requestMapConfig) {
|
||||
assert.ifError(err);
|
||||
self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this);
|
||||
},
|
||||
function prepareAnalysisLayers(err, requestMapConfig) {
|
||||
assert.ifError(err);
|
||||
var analysisConfiguration = {
|
||||
@ -142,7 +148,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
|
||||
assert.ifError(err);
|
||||
var next = this;
|
||||
|
||||
self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, function(err, layers) {
|
||||
self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, self.analysesResults, function(err, layers) {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
|
@ -1,13 +1,15 @@
|
||||
var step = require('step');
|
||||
var queue = require('queue-async');
|
||||
var _ = require('underscore');
|
||||
|
||||
function MapConfigOverviewsAdapter(overviewsMetadataApi) {
|
||||
function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) {
|
||||
this.overviewsMetadataApi = overviewsMetadataApi;
|
||||
this.filterStatsApi = filterStatsApi;
|
||||
}
|
||||
|
||||
module.exports = MapConfigOverviewsAdapter;
|
||||
|
||||
MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) {
|
||||
MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analysesResults, callback) {
|
||||
var self = this;
|
||||
|
||||
if (!layers || layers.length === 0) {
|
||||
@ -24,11 +26,52 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callb
|
||||
if (err) {
|
||||
done(err, layer);
|
||||
} else {
|
||||
if ( !_.isEmpty(metadata) ) {
|
||||
layer = _.extend({}, layer);
|
||||
layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } });
|
||||
}
|
||||
done(null, layer);
|
||||
var query_rewrite_data = { overviews: metadata };
|
||||
step(
|
||||
function collectFiltersData() {
|
||||
var filters, unfiltered_query;
|
||||
if ( layer.options.source && analysesResults ) {
|
||||
var sourceId = layer.options.source.id;
|
||||
var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
|
||||
if ( node ) {
|
||||
node = node.rootNode;
|
||||
filters = node.filters; // TODO: node.getFilters() when available in camshaft
|
||||
var filters_disabler = Object.keys(filters).reduce(
|
||||
function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
|
||||
{}
|
||||
);
|
||||
unfiltered_query = node.getQuery(filters_disabler);
|
||||
query_rewrite_data.filters = filters;
|
||||
query_rewrite_data.unfiltered_query = unfiltered_query;
|
||||
}
|
||||
}
|
||||
this(null, filters, unfiltered_query);
|
||||
},
|
||||
function collectStatsData(err, filters, unfiltered_query) {
|
||||
var next_step = this;
|
||||
if ( filters ) {
|
||||
self.filterStatsApi.getFilterStats(
|
||||
username,
|
||||
unfiltered_query, filters,
|
||||
function(err, stats) {
|
||||
if ( !err ) {
|
||||
query_rewrite_data.filter_stats = stats;
|
||||
}
|
||||
return next_step(err);
|
||||
}
|
||||
);
|
||||
} else {
|
||||
return next_step(null);
|
||||
}
|
||||
},
|
||||
function addDataToLayer(err) {
|
||||
if ( !err && !_.isEmpty(metadata) ) {
|
||||
layer = _.extend({}, layer);
|
||||
layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data });
|
||||
}
|
||||
done(err, layer);
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
@ -21,6 +21,7 @@ var mapnik = windshaft.mapnik;
|
||||
|
||||
var TemplateMaps = require('./backends/template_maps.js');
|
||||
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
|
||||
var FilterStatsApi = require('./api/filter_stats_api');
|
||||
var UserLimitsApi = require('./api/user_limits_api');
|
||||
var AuthApi = require('./api/auth_api');
|
||||
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
|
||||
@ -59,6 +60,7 @@ module.exports = function(serverOptions) {
|
||||
var pgConnection = new PgConnection(metadataBackend);
|
||||
var pgQueryRunner = new PgQueryRunner(pgConnection);
|
||||
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
|
||||
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
|
||||
var userLimitsApi = new UserLimitsApi(metadataBackend, {
|
||||
limits: {
|
||||
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
|
||||
@ -148,7 +150,7 @@ module.exports = function(serverOptions) {
|
||||
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
|
||||
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
|
||||
|
||||
var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
|
||||
var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
|
||||
|
||||
var turboCartoParser = new TurboCartoParser(pgQueryRunner);
|
||||
var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser);
|
||||
|
@ -1,5 +1,26 @@
|
||||
var _ = require('underscore');
|
||||
var TableNameParser = require('./table_name_parser');
|
||||
|
||||
var BBoxFilter = require('../models/filter/bbox');
|
||||
var CamshaftFilter = require('../models/filter/camshaft');
|
||||
|
||||
// Minimim number of filtered rows to use overviews
|
||||
var FILTER_MIN_ROWS = 65536;
|
||||
// Maximum filtered fraction to not apply overviews
|
||||
var FILTER_MAX_FRACTION = 0.2;
|
||||
|
||||
function apply_filters_to_query(query, filters, bbox_filter) {
|
||||
if ( filters && !_.isEmpty(filters)) {
|
||||
var camshaftFilter = new CamshaftFilter(filters);
|
||||
query = camshaftFilter.sql(query);
|
||||
}
|
||||
if ( bbox_filter ) {
|
||||
var bboxFilter = new BBoxFilter(bbox_filter.options, bbox_filter.params);
|
||||
query = bboxFilter.sql(query);
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
function OverviewsQueryRewriter(options) {
|
||||
|
||||
this.options = options;
|
||||
@ -119,26 +140,34 @@ function replace_table_in_query(sql, old_table_name, replacement) {
|
||||
return sql.replace(new RegExp(regexp, 'g'), replacement);
|
||||
}
|
||||
|
||||
function overviews_query(query, overviews, zoom_level_expression) {
|
||||
|
||||
function replace_table_in_query_with_schema(query, table, schema, replacement) {
|
||||
if ( replacement ) {
|
||||
query = replace_table_in_query(query, table, replacement);
|
||||
var parsed_table = TableNameParser.parse(table);
|
||||
if (!parsed_table.schema && schema) {
|
||||
// replace also the qualified table name, if the table wasn't qualified
|
||||
parsed_table.schema = schema;
|
||||
table = TableNameParser.table_identifier(parsed_table);
|
||||
query = replace_table_in_query(query, table, replacement);
|
||||
}
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
// Build query to use overviews for a variant zoom level (given by a expression to
|
||||
// be evaluated by the database server)
|
||||
function overviews_query_with_zoom_expression(query, overviews, zoom_level_expression) {
|
||||
var replaced_query = query;
|
||||
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
|
||||
var replacement;
|
||||
for ( var table in overviews ) {
|
||||
if (overviews.hasOwnProperty(table)) {
|
||||
var table_overviews = overviews[table];
|
||||
var table_view = overviews_view_name(table);
|
||||
var schema = table_overviews.schema;
|
||||
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
|
||||
replaced_query = replace_table_in_query(replaced_query, table, replacement);
|
||||
var parsed_table = TableNameParser.parse(table);
|
||||
if (!parsed_table.schema && schema) {
|
||||
// replace also the qualified table name, if the table wasn't qualified
|
||||
parsed_table.schema = schema;
|
||||
table = TableNameParser.table_identifier(parsed_table);
|
||||
replaced_query = replace_table_in_query(replaced_query, table, replacement);
|
||||
}
|
||||
}
|
||||
}
|
||||
_.each(Object.keys(overviews), function(table) {
|
||||
var table_overviews = overviews[table];
|
||||
var table_view = overviews_view_name(table);
|
||||
var schema = table_overviews.schema;
|
||||
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
|
||||
replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement);
|
||||
});
|
||||
if ( replaced_query !== query ) {
|
||||
sql += "\n";
|
||||
sql += replaced_query;
|
||||
@ -148,34 +177,128 @@ function overviews_query(query, overviews, zoom_level_expression) {
|
||||
return sql;
|
||||
}
|
||||
|
||||
// Build query to use overviews for a specific zoom level value
|
||||
function overviews_query_with_definite_zoom(query, overviews, zoom_level) {
|
||||
var replaced_query = query;
|
||||
var replacement;
|
||||
_.each(Object.keys(overviews), function(table) {
|
||||
var table_overviews = overviews[table];
|
||||
var schema = table_overviews.schema;
|
||||
replacement = overview_table_for_zoom_level(table_overviews, zoom_level);
|
||||
replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement);
|
||||
});
|
||||
return replaced_query;
|
||||
}
|
||||
|
||||
// Find a suitable overview table for a specific zoom_level
|
||||
function overview_table_for_zoom_level(table_overviews, zoom_level) {
|
||||
var overview_table;
|
||||
if ( table_overviews ) {
|
||||
overview_table = table_overviews[zoom_level];
|
||||
if ( !overview_table ) {
|
||||
_.every(Object.keys(table_overviews).sort(function(x,y){ return x-y; }), function(overview_zoom) {
|
||||
if ( +overview_zoom > +zoom_level ) {
|
||||
overview_table = table_overviews[overview_zoom];
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
if ( overview_table ) {
|
||||
overview_table = overview_table.table;
|
||||
}
|
||||
return overview_table;
|
||||
}
|
||||
|
||||
// Transform an SQL query so that it uses overviews.
|
||||
// overviews contains metadata about the overviews to be used:
|
||||
// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
|
||||
//
|
||||
// For a given query `SELECT * FROM table`, if any of tables in it
|
||||
// has overviews as defined by the provided metadat, the query will
|
||||
// be transform into something similar to this:
|
||||
//
|
||||
// WITH _vovw_scale AS ( ... ), -- define scale level
|
||||
// WITH _vovw_table AS ( ... ), -- define union of overviews and base table
|
||||
// SELECT * FROM _vovw_table -- query with table replaced by _vovw_table
|
||||
// SELECT * FROM -- in the query the table is replaced by:
|
||||
// ( ... ) AS _vovw_table -- a union of overviews and base table
|
||||
//
|
||||
// This transformation can in principle be applied to arbitrary queries
|
||||
// (except for the case of queries that include the name of tables with
|
||||
// overviews inside text literals: at the current table name substitution
|
||||
// doesnn't prevent substitution inside literals).
|
||||
// But the transformation will currently only be applied to simple queries
|
||||
// of the form detected by the overviews_supported_query function.
|
||||
OverviewsQueryRewriter.prototype.query = function(query, data) {
|
||||
var overviews = this.overviews_metadata(data);
|
||||
if ( !overviews || !this.is_supported_query(query)) {
|
||||
// The data argument has the form:
|
||||
// {
|
||||
// overviews: // overview tables metadata
|
||||
// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... },
|
||||
// zoom_level: ..., // optional zoom level
|
||||
// filters: ..., // filters definition
|
||||
// unfiltered_query: ..., // query without the filters
|
||||
// bbox_filter: ... // bounding-box filter
|
||||
// }
|
||||
OverviewsQueryRewriter.prototype.query = function(query, data, options) {
|
||||
options = options || {};
|
||||
data = data || {};
|
||||
|
||||
var overviews = data.overviews;
|
||||
var unfiltered_query = data.unfiltered_query;
|
||||
var filters = data.filters;
|
||||
var bbox_filter = data.bbox_filter;
|
||||
|
||||
if ( !unfiltered_query ) {
|
||||
unfiltered_query = query;
|
||||
}
|
||||
|
||||
if ( !should_use_overviews(unfiltered_query, data) ) {
|
||||
return query;
|
||||
}
|
||||
var zoom_level_expression = this.options.zoom_level || '0';
|
||||
return overviews_query(query, overviews, zoom_level_expression);
|
||||
|
||||
var rewritten_query;
|
||||
|
||||
var zoom_level_expression = this.options.zoom_level;
|
||||
var zoom_level = zoom_level_for_query(unfiltered_query, zoom_level_expression, options);
|
||||
|
||||
rewritten_query = overviews_query(unfiltered_query, overviews, zoom_level, zoom_level_expression);
|
||||
|
||||
if ( rewritten_query === unfiltered_query ) {
|
||||
// could not or didn't need to alter the query
|
||||
rewritten_query = query;
|
||||
} else {
|
||||
rewritten_query = apply_filters_to_query(rewritten_query, filters, bbox_filter);
|
||||
}
|
||||
|
||||
return rewritten_query;
|
||||
};
|
||||
|
||||
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
|
||||
function zoom_level_for_query(query, zoom_level_expression, options) {
|
||||
var zoom_level = null;
|
||||
if ( _.has(options, 'zoom_level') ) {
|
||||
zoom_level = options.zoom_level || '0';
|
||||
}
|
||||
if ( zoom_level === null && !zoom_level_expression ) {
|
||||
zoom_level = '0';
|
||||
}
|
||||
return zoom_level;
|
||||
}
|
||||
|
||||
function overviews_query(query, overviews, zoom_level, zoom_level_expression) {
|
||||
if ( zoom_level || zoom_level === '0' || zoom_level === 0 ) {
|
||||
return overviews_query_with_definite_zoom(query, overviews, zoom_level);
|
||||
} else {
|
||||
return overviews_query_with_zoom_expression(query, overviews, zoom_level_expression);
|
||||
}
|
||||
}
|
||||
|
||||
function should_use_overviews(query, data) {
|
||||
data = data || {};
|
||||
var use_overviews = data.overviews && is_supported_query(query);
|
||||
if ( use_overviews && data.filters && data.filter_stats ) {
|
||||
var filtered_rows = data.filter_stats.filtered_rows;
|
||||
var unfiltered_rows = data.filter_stats.unfiltered_rows;
|
||||
if ( unfiltered_rows && (filtered_rows || filtered_rows === 0) ) {
|
||||
use_overviews = filtered_rows >= FILTER_MIN_ROWS ||
|
||||
(filtered_rows/unfiltered_rows) > FILTER_MAX_FRACTION;
|
||||
}
|
||||
}
|
||||
return use_overviews;
|
||||
}
|
||||
|
||||
function is_supported_query(sql) {
|
||||
var basic_query =
|
||||
/\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
|
||||
var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i');
|
||||
@ -187,8 +310,4 @@ OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
|
||||
'i'
|
||||
);
|
||||
return !!(sql.match(unwrapped_query) || sql.match(wrapped_query));
|
||||
};
|
||||
|
||||
OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {
|
||||
return data && data.overviews;
|
||||
};
|
||||
}
|
||||
|
2
npm-shrinkwrap.json
generated
2
npm-shrinkwrap.json
generated
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "windshaft-cartodb",
|
||||
"version": "2.42.3",
|
||||
"version": "2.44.0",
|
||||
"dependencies": {
|
||||
"body-parser": {
|
||||
"version": "1.14.2",
|
||||
|
@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": true,
|
||||
"name": "windshaft-cartodb",
|
||||
"version": "2.42.3",
|
||||
"version": "2.44.0",
|
||||
"description": "A map tile server for CartoDB",
|
||||
"keywords": [
|
||||
"cartodb"
|
||||
|
358
test/acceptance/dataviews/overviews.js
Normal file
358
test/acceptance/dataviews/overviews.js
Normal file
@ -0,0 +1,358 @@
|
||||
require('../../support/test_helper');
|
||||
|
||||
var assert = require('../../support/assert');
|
||||
var TestClient = require('../../support/test-client');
|
||||
|
||||
describe('dataviews using tables without overviews', function() {
|
||||
|
||||
var nonOverviewsMapConfig = {
|
||||
version: '1.5.0',
|
||||
analyses: [
|
||||
{ id: 'data-source',
|
||||
type: 'source',
|
||||
params: {
|
||||
query: 'select * from populated_places_simple_reduced'
|
||||
}
|
||||
}
|
||||
],
|
||||
dataviews: {
|
||||
country_places_count: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'adm0_a3',
|
||||
operation: 'count'
|
||||
}
|
||||
},
|
||||
country_categories: {
|
||||
type: 'aggregation',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'adm0_a3',
|
||||
aggregation: 'count'
|
||||
}
|
||||
}
|
||||
},
|
||||
layers: [
|
||||
{
|
||||
type: 'mapnik',
|
||||
options: {
|
||||
sql: 'select * from populated_places_simple_reduced',
|
||||
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
|
||||
cartocss_version: '2.3.0',
|
||||
source: { id: 'data-source' }
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
it("should expose a formula", function(done) {
|
||||
var testClient = new TestClient(nonOverviewsMapConfig);
|
||||
testClient.getDataview('country_places_count', { own_filter: 0 }, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, { operation: 'count', result: 7313, nulls: 0, type: 'formula' });
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should admit a bbox", function(done) {
|
||||
var params = {
|
||||
bbox: "-170,-80,170,80"
|
||||
};
|
||||
var testClient = new TestClient(nonOverviewsMapConfig);
|
||||
testClient.getDataview('country_places_count', params, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, { operation: 'count', result: 7253, nulls: 0, type: 'formula' });
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filters', function() {
|
||||
|
||||
describe('category', function () {
|
||||
|
||||
it("should expose a filtered formula", function (done) {
|
||||
var params = {
|
||||
filters: {
|
||||
dataviews: {country_categories: {accept: ['CAN']}}
|
||||
}
|
||||
};
|
||||
var testClient = new TestClient(nonOverviewsMapConfig);
|
||||
testClient.getDataview('country_places_count', params, function (err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, { operation: 'count', result: 256, nulls: 0, type: 'formula' });
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a filtered formula and admit a bbox", function (done) {
|
||||
var params = {
|
||||
filters: {
|
||||
dataviews: {country_categories: {accept: ['CAN']}}
|
||||
},
|
||||
bbox: "-170,-80,170,80"
|
||||
};
|
||||
var testClient = new TestClient(nonOverviewsMapConfig);
|
||||
testClient.getDataview('country_places_count', params, function (err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, { operation: 'count', result: 254, nulls: 0, type: 'formula' });
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
describe('dataviews using tables with overviews', function() {
|
||||
|
||||
var overviewsMapConfig = {
|
||||
version: '1.5.0',
|
||||
analyses: [
|
||||
{ id: 'data-source',
|
||||
type: 'source',
|
||||
params: {
|
||||
query: 'select * from test_table_overviews'
|
||||
}
|
||||
}
|
||||
],
|
||||
dataviews: {
|
||||
test_sum: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'value',
|
||||
operation: 'sum'
|
||||
}
|
||||
},
|
||||
test_categories: {
|
||||
type: 'aggregation',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'name',
|
||||
aggregation: 'count',
|
||||
aggregationColumn: 'name',
|
||||
}
|
||||
},
|
||||
test_avg: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'value',
|
||||
operation: 'avg'
|
||||
}
|
||||
},
|
||||
test_count: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'value',
|
||||
operation: 'count'
|
||||
}
|
||||
},
|
||||
test_min: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'value',
|
||||
operation: 'min'
|
||||
}
|
||||
},
|
||||
test_max: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
options: {
|
||||
column: 'value',
|
||||
operation: 'max'
|
||||
}
|
||||
}
|
||||
},
|
||||
layers: [
|
||||
{
|
||||
type: 'mapnik',
|
||||
options: {
|
||||
sql: 'select * from test_table_overviews',
|
||||
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
|
||||
cartocss_version: '2.3.0',
|
||||
source: { id: 'data-source' }
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
it("should expose a sum formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_sum', { own_filter: 0 }, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose an avg formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_avg', { own_filter: 0 }, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a count formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_count', { own_filter: 0 }, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a max formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_max', { own_filter: 0 }, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a min formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_min', { own_filter: 0 }, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should admit a bbox", function(done) {
|
||||
var params = {
|
||||
bbox: "-170,-80,170,80"
|
||||
};
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_sum', params, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filters', function() {
|
||||
|
||||
describe('category', function () {
|
||||
|
||||
var params = {
|
||||
filters: {
|
||||
dataviews: {test_categories: {accept: ['Hawai']}}
|
||||
}
|
||||
};
|
||||
|
||||
it("should expose a filtered sum formula", function (done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_sum', params, function (err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a filtered avg formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_avg', params, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a filtered count formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_count', params, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a filterd max formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_max', params, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a filterd min formula", function(done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_min', params, function(err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it("should expose a filtered sum formula with bbox", function (done) {
|
||||
var bboxparams = {
|
||||
filters: {
|
||||
dataviews: {test_categories: {accept: ['Hawai']}}
|
||||
},
|
||||
bbox: "-170,-80,170,80"
|
||||
};
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_sum', bboxparams, function (err, formula_result) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
});
|
@ -109,3 +109,118 @@ describe('overviews metadata', function() {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('overviews metadata with filters', function() {
|
||||
// configure redis pool instance to use in tests
|
||||
var redisPool = new RedisPool(global.environment.redis);
|
||||
|
||||
var keysToDelete;
|
||||
|
||||
beforeEach(function() {
|
||||
keysToDelete = {};
|
||||
});
|
||||
|
||||
afterEach(function(done) {
|
||||
test_helper.deleteRedisKeys(keysToDelete, done);
|
||||
});
|
||||
|
||||
it("layers with overviews", function(done) {
|
||||
|
||||
var layergroup = {
|
||||
version: '1.5.0',
|
||||
layers: [
|
||||
{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: 'SELECT * FROM test_table_overviews',
|
||||
source: { id: 'with_overviews' },
|
||||
cartocss: '#layer { marker-fill: black; }',
|
||||
cartocss_version: '2.3.0'
|
||||
}
|
||||
}
|
||||
],
|
||||
dataviews: {
|
||||
test_names: {
|
||||
type: 'aggregation',
|
||||
source: {id: 'with_overviews'},
|
||||
options: {
|
||||
column: 'name',
|
||||
aggregation: 'count'
|
||||
}
|
||||
}
|
||||
},
|
||||
analyses: [
|
||||
{ id: 'with_overviews',
|
||||
type: 'source',
|
||||
params: {
|
||||
query: 'select * from test_table_overviews'
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
var filters = {
|
||||
dataviews: {
|
||||
test_names: { accept: ['Hawai'] }
|
||||
}
|
||||
};
|
||||
|
||||
var layergroup_url = '/api/v1/map';
|
||||
|
||||
var expected_token;
|
||||
step(
|
||||
function do_post()
|
||||
{
|
||||
var next = this;
|
||||
assert.response(server, {
|
||||
url: layergroup_url + '?filters=' + JSON.stringify(filters),
|
||||
method: 'POST',
|
||||
headers: {host: 'localhost', 'Content-Type': 'application/json' },
|
||||
data: JSON.stringify(layergroup)
|
||||
}, {}, function(res) {
|
||||
assert.equal(res.statusCode, 200, res.body);
|
||||
var parsedBody = JSON.parse(res.body);
|
||||
assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
|
||||
expected_token = parsedBody.layergroupid;
|
||||
next(null, res);
|
||||
});
|
||||
},
|
||||
function do_get_mapconfig(err)
|
||||
{
|
||||
assert.ifError(err);
|
||||
var next = this;
|
||||
|
||||
var mapStore = new windshaft.storage.MapStore({
|
||||
pool: redisPool,
|
||||
expire_time: 500000
|
||||
});
|
||||
mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) {
|
||||
assert.ifError(err);
|
||||
assert.equal(mapConfig._cfg.layers[0].type, 'cartodb');
|
||||
assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data);
|
||||
var expected_data = {
|
||||
overviews: {
|
||||
test_table_overviews: {
|
||||
schema: 'public',
|
||||
1: { table: '_vovw_1_test_table_overviews' },
|
||||
2: { table: '_vovw_2_test_table_overviews' }
|
||||
}
|
||||
},
|
||||
filters: { test_names: { type: 'category', column: 'name', params: { accept: [ 'Hawai' ] } } },
|
||||
unfiltered_query: 'select * from test_table_overviews',
|
||||
filter_stats: { unfiltered_rows: 5, filtered_rows: 1 }
|
||||
};
|
||||
assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data);
|
||||
|
||||
});
|
||||
|
||||
next(err);
|
||||
},
|
||||
function finish(err) {
|
||||
keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
|
||||
keysToDelete['user:localhost:mapviews:global'] = 5;
|
||||
done(err);
|
||||
}
|
||||
);
|
||||
});
|
||||
});
|
||||
|
52
test/acceptance/sql-wrap.js
Normal file
52
test/acceptance/sql-wrap.js
Normal file
@ -0,0 +1,52 @@
|
||||
require('../support/test_helper');
|
||||
|
||||
var assert = require('../support/assert');
|
||||
var TestClient = require('../support/test-client');
|
||||
|
||||
describe('sql-wrap', function() {
|
||||
|
||||
afterEach(function(done) {
|
||||
if (this.testClient) {
|
||||
this.testClient.drain(done);
|
||||
} else {
|
||||
return done();
|
||||
}
|
||||
});
|
||||
|
||||
it('should use sql_wrap from layer options', function(done) {
|
||||
var mapConfig = {
|
||||
version: '1.5.0',
|
||||
layers: [
|
||||
{
|
||||
"type": "cartodb",
|
||||
"options": {
|
||||
"sql": "SELECT * FROM populated_places_simple_reduced",
|
||||
"sql_wrap": "SELECT * FROM (<%= sql %>) _w WHERE adm0_a3 = 'USA'",
|
||||
"cartocss": [
|
||||
"#points {",
|
||||
" marker-fill-opacity: 1;",
|
||||
" marker-line-color: #FFF;",
|
||||
" marker-line-width: 0.5;",
|
||||
" marker-line-opacity: 1;",
|
||||
" marker-placement: point;",
|
||||
" marker-type: ellipse;",
|
||||
" marker-width: 8;",
|
||||
" marker-fill: red;",
|
||||
" marker-allow-overlap: true;",
|
||||
"}"
|
||||
].join('\n'),
|
||||
"cartocss_version": "2.3.0"
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
this.testClient = new TestClient(mapConfig, 1234);
|
||||
this.testClient.getTile(0, 0, 0, function(err, tile, img) {
|
||||
assert.ok(!err, err);
|
||||
var fixtureImg = './test/fixtures/sql-wrap-usa-filter.png';
|
||||
assert.imageIsSimilarToFile(img, fixtureImg, 20, done);
|
||||
});
|
||||
});
|
||||
|
||||
});
|
BIN
test/fixtures/sql-wrap-usa-filter.png
vendored
Normal file
BIN
test/fixtures/sql-wrap-usa-filter.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 4.1 KiB |
@ -6,6 +6,7 @@ var cartodbRedis = require('cartodb-redis');
|
||||
var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
|
||||
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
|
||||
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
|
||||
var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api');
|
||||
var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter');
|
||||
|
||||
// configure redis pool instance to use in tests
|
||||
@ -17,9 +18,9 @@ var metadataBackend = cartodbRedis({pool: redisPool});
|
||||
var pgConnection = new PgConnection(metadataBackend);
|
||||
var pgQueryRunner = new PgQueryRunner(pgConnection);
|
||||
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
|
||||
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
|
||||
|
||||
|
||||
var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
|
||||
var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
|
||||
|
||||
describe('MapConfigOverviewsAdapter', function() {
|
||||
|
||||
@ -36,7 +37,7 @@ describe('MapConfigOverviewsAdapter', function() {
|
||||
}
|
||||
};
|
||||
|
||||
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
|
||||
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
|
||||
assert.ok(!err);
|
||||
assert.equal(layers.length, 1);
|
||||
assert.equal(layers[0].type, 'cartodb');
|
||||
@ -64,7 +65,7 @@ describe('MapConfigOverviewsAdapter', function() {
|
||||
}
|
||||
};
|
||||
|
||||
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
|
||||
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
|
||||
assert.ok(!err);
|
||||
assert.equal(layers.length, 1);
|
||||
assert.equal(layers[0].type, 'cartodb');
|
||||
|
@ -247,8 +247,10 @@ CREATE TABLE test_table_overviews (
|
||||
cartodb_id integer NOT NULL,
|
||||
name character varying,
|
||||
address character varying,
|
||||
value float8,
|
||||
the_geom geometry,
|
||||
the_geom_webmercator geometry,
|
||||
_feature_count integer,
|
||||
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
|
||||
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
|
||||
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
|
||||
@ -274,11 +276,11 @@ SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true);
|
||||
ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass);
|
||||
|
||||
INSERT INTO test_table_overviews VALUES
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 3.0, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 5.0, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1);
|
||||
|
||||
ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id);
|
||||
|
||||
@ -294,8 +296,10 @@ CREATE TABLE _vovw_1_test_table_overviews (
|
||||
cartodb_id integer NOT NULL,
|
||||
name character varying,
|
||||
address character varying,
|
||||
value float8,
|
||||
the_geom geometry,
|
||||
the_geom_webmercator geometry,
|
||||
_feature_count integer,
|
||||
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
|
||||
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
|
||||
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
|
||||
@ -313,8 +317,10 @@ CREATE TABLE _vovw_2_test_table_overviews (
|
||||
cartodb_id integer NOT NULL,
|
||||
name character varying,
|
||||
address character varying,
|
||||
value float8,
|
||||
the_geom geometry,
|
||||
the_geom_webmercator geometry,
|
||||
_feature_count integer,
|
||||
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
|
||||
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
|
||||
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
|
||||
@ -327,11 +333,11 @@ GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER;
|
||||
GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER;
|
||||
|
||||
INSERT INTO _vovw_2_test_table_overviews VALUES
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 8.0/3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 3),
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 7.0/2.0, '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 2);
|
||||
|
||||
INSERT INTO _vovw_1_test_table_overviews VALUES
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241');
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5);
|
||||
|
||||
|
||||
-- analysis tables -----------------------------------------------
|
||||
|
@ -475,4 +475,137 @@ describe('Overviews query rewriter', function() {
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
});
|
||||
|
||||
it('generates query for specific Z level', function(){
|
||||
var sql = "SELECT * FROM table1";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
0: { table: 'table1_ov0' },
|
||||
2: { table: 'table1_ov2' },
|
||||
3: { table: 'table1_ov3' }
|
||||
}
|
||||
}
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 3 });
|
||||
var expected_sql = "SELECT * FROM table1_ov3";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
});
|
||||
|
||||
it('generates query for specific nonpresent Z level', function(){
|
||||
var sql = "SELECT * FROM table1";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
0: { table: 'table1_ov0' },
|
||||
2: { table: 'table1_ov2' },
|
||||
3: { table: 'table1_ov3' }
|
||||
}
|
||||
}
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 1 });
|
||||
var expected_sql = "SELECT * FROM table1_ov2";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
});
|
||||
|
||||
it('does not use overviews for specific out-of-range Z level', function(){
|
||||
var sql = "SELECT * FROM table1";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
0: { table: 'table1_ov0' },
|
||||
2: { table: 'table1_ov2' },
|
||||
3: { table: 'table1_ov3' }
|
||||
}
|
||||
}
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 4 });
|
||||
var expected_sql = "SELECT * FROM table1";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
});
|
||||
|
||||
it('generates query with filters', function(){
|
||||
var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
|
||||
FROM (SELECT *\
|
||||
FROM (select * from table1) _camshaft_category_filter\
|
||||
WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
0: { table: 'table1_ov0' },
|
||||
1: { table: 'table1_ov1' },
|
||||
2: { table: 'table1_ov2' },
|
||||
3: { table: 'table1_ov3' }
|
||||
}
|
||||
},
|
||||
filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
|
||||
unfiltered_query: 'SELECT * FROM table1'
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
SELECT * FROM (WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
|
||||
) AS _vovw_table1) _camshaft_category_filter\
|
||||
WHERE name IN ($escape_0$X$escape_0$)\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
});
|
||||
|
||||
it('generates query with filters for specific zoom level', function(){
|
||||
var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
|
||||
FROM (SELECT *\
|
||||
FROM (select * from table1) _camshaft_category_filter\
|
||||
WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
0: { table: 'table1_ov0' },
|
||||
1: { table: 'table1_ov1' },
|
||||
2: { table: 'table1_ov2' },
|
||||
3: { table: 'table1_ov3' }
|
||||
}
|
||||
},
|
||||
filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
|
||||
unfiltered_query: 'SELECT * FROM table1',
|
||||
filter_stats: { unfiltered_rows: 1000, filtered_rows: 900 }
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 2 });
|
||||
var expected_sql = "\
|
||||
SELECT * FROM (SELECT * FROM table1_ov2) _camshaft_category_filter\
|
||||
WHERE name IN ($escape_0$X$escape_0$)\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
});
|
||||
|
||||
it('does not generates query with aggressive filtering', function(){
|
||||
var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
|
||||
FROM (SELECT *\
|
||||
FROM (select * from table1) _camshaft_category_filter\
|
||||
WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
0: { table: 'table1_ov0' },
|
||||
1: { table: 'table1_ov1' },
|
||||
2: { table: 'table1_ov2' },
|
||||
3: { table: 'table1_ov3' }
|
||||
}
|
||||
},
|
||||
filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
|
||||
unfiltered_query: 'SELECT * FROM table1',
|
||||
filter_stats: { unfiltered_rows: 1000, filtered_rows: 10 }
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
assert.equal(overviews_sql, sql);
|
||||
});
|
||||
});
|
||||
|
Loading…
Reference in New Issue
Block a user