cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP: fix problems for aggregations & metadata

Browse Source
This commit is contained in:
Javier Goizueta 2018-05-10 19:12:47 +02:00
parent f7745928ab
commit cae4dd81c9
3 changed files with 36 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
lib/cartodb/backends/layer-stats/mapnik-layer-stats.js
View File

@ -9,6 +9,8 @@ function MapnikLayerStats () {
};
}
// TODO: estimatedFeatureCount is post-aggregation; the rest is pre; distinguish and complement
MapnikLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
@ -41,7 +43,7 @@ function columnAggregations(field) {
function firstPhaseQueries(queries, ctx) {
if (queries.results.estimatedFeatureCount === undefined) {
queries.task(
queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.query), function(err, res) {
queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.aggrQuery), function(err, res) {
if (err) {
// at least for debugging we should err
queries.results.estimatedFeatureCount = -1;
@ -56,11 +58,13 @@ function firstPhaseQueries(queries, ctx) {
}
if (ctx.metaOptions.featureCount) {
// TODO: pre/aggr
// TODO: for pre, use ctx.aggrMeta.pre_aggregation_count
// TODO: if ctx.metaOptions.columnStats we can combine this with column stats query
queries.task(
queryPromise(
ctx.dbConnection,
queryUtils.getQueryActualRowCount(ctx.rawQuery),
queryUtils.getQueryActualRowCount(ctx.preQuery),
(err, res) => {
if (err) {
queries.results.featureCount = -1;
@ -74,11 +78,13 @@ function firstPhaseQueries(queries, ctx) {
}
if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) {
// TODO: pre/aggr
// TODO: for pre, use ctx.aggrMeta.geometry_type
const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn();
queries.task(
queryPromise(
ctx.dbConnection,
queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn),
queryUtils.getQueryGeometryType(ctx.preQuery, geometryColumn),
(err, res) => {
if (!err) {
queries.results.geometryType = res.rows[0].geom_type;
@ -90,12 +96,15 @@ function firstPhaseQueries(queries, ctx) {
}
if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) {
// TODO: pre/aggr
// TODO: for aggr, use layer.options.columns (will need to pass in ctx)
// note: post-aggregation columns are in layer.options.columns when aggregation is present
queries.task(
// TODO: note we have getLayerColumns in aggregation mapconfig.
// and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters
queryPromise(
ctx.dbConnection,
queryUtils.getQueryLimited(ctx.rawQuery, 0),
queryUtils.getQueryLimited(ctx.preQuery, 0),
(err, res) => {
if (!err) {
queries.results.columns = formatResultFields(ctx.dbConnection, res.fields);
@ -116,7 +125,7 @@ function secondPhaseQueries(queries, ctx) {
queries.task(
queryPromise(
ctx.dbConnection,
queryUtils.getQuerySample(ctx.rawQuery, sampleProb),
queryUtils.getQuerySample(ctx.preQuery, sampleProb),
(err, res) => {
if (err) {
queries.results.sample = [];
@ -130,6 +139,7 @@ function secondPhaseQueries(queries, ctx) {
}
if (ctx.metaOptions.columnStats) {
// TODO: pre/aggr
let aggr = [];
Object.keys(queries.results.columns).forEach(name => {
aggr = aggr.concat(
@ -143,7 +153,7 @@ function secondPhaseQueries(queries, ctx) {
queries.task(
queryPromise(
ctx.dbConnection,
queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN),
queryUtils.getQueryTopCategories(ctx.preQuery, name, topN),
(err, res) => {
if (!err) {
queries.results.columns[name].categories = res.rows;
@ -157,7 +167,7 @@ function secondPhaseQueries(queries, ctx) {
queries.task(
queryPromise(
ctx.dbConnection,
`SELECT ${aggr.join(',')} FROM (${ctx.rawQuery}) AS __cdb_query`,
`SELECT ${aggr.join(',')} FROM (${ctx.preQuery}) AS __cdb_query`,
(err, res) => {
if (!err) {
Object.keys(queries.results.columns).forEach(name => {
@ -218,10 +228,14 @@ function formatResultFields(dbConnection, flds) {
MapnikLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
let aggrQuery = layer.options.sql_raw || layer.options.sql;
let preQuery = layer.options.aggregation_metadata ? layer.options.aggregation_metadata.pre_aggregation_sql : aggrQuery;
let context = {
dbConnection,
query: layer.options.sql,
rawQuery: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql,
preQuery,
aggrQuery,
aggrMeta: layer.options.aggregation_metadata,
metaOptions: layer.options.metadata || {}
};

14
lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js
View File

@ -73,7 +73,7 @@ module.exports = class AggregationMapConfigAdapter {
if (adapted) {
requestMapConfig.layers[index] = layer;
}
const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted);
const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); // <<-
context.aggregation.layers.push(aggregatedFormats);
});
@ -84,7 +84,7 @@ module.exports = class AggregationMapConfigAdapter {
_adaptLayer (connection, mapConfig, layer, index) {
return new Promise((resolve, reject) => {
this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt) => {
this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt, aggrMeta) => {
if (err) {
return reject(err);
}
@ -93,6 +93,7 @@ module.exports = class AggregationMapConfigAdapter {
return resolve({ layer, index, adapted: shouldAdapt });
}
const sqlQuery = layer.options.sql;
const sqlQueryWrap = layer.options.sql_wrap;
let aggregationSql = mapConfig.getAggregatedQuery(index);
@ -110,6 +111,12 @@ module.exports = class AggregationMapConfigAdapter {
layer.options.columns = columns;
layer.options.aggregation_metadata = {
pre_aggregation_sql: sqlQueryWrap || sqlQuery,
geometry_type: aggrMeta.type,
pre_aggregation_count: aggrMeta.count
};
return resolve({ layer, index, adapted: shouldAdapt });
});
});
@ -154,11 +161,12 @@ module.exports = class AggregationMapConfigAdapter {
return callback(null, false);
}
callback(null, true);
callback(null, true, result);
});
}
_getAggregationMetadata (mapConfig, layer, adapted) {
// also: pre-aggr query, columns, ...
if (!adapted) {
return { png: false, mvt: false };
}

2
test/acceptance/stats/mapnik_stats_layergroup.js
View File

@ -474,4 +474,6 @@ describe('Create mapnik layergroup', function() {
testClient.drain(done);
});
});
// TODO: add tests for metadata with aggregation
});