Draft
This commit is contained in:
Daniel García Aubert
parent
f82e403180
commit
c3df075d91
@ -21,6 +21,8 @@ const OverviewsMetadataBackend = require('../backends/overviews-metadata');
|
||||
const FilterStatsApi = require('../backends/filter-stats');
|
||||
const TablesExtentBackend = require('../backends/tables-extent');
|
||||
|
||||
const ClusterBackend = require('../backends/cluster');
|
||||
|
||||
const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables');
|
||||
const SurrogateKeysCache = require('../cache/surrogate_keys_cache');
|
||||
const VarnishHttpCacheBackend = require('../cache/backend/varnish_http');
|
||||
@ -110,6 +112,7 @@ module.exports = class ApiRouter {
|
||||
const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
|
||||
const dataviewBackend = new DataviewBackend(analysisBackend);
|
||||
const statsBackend = new StatsBackend();
|
||||
const clusterBackend = new ClusterBackend();
|
||||
|
||||
const userLimitsBackend = new UserLimitsBackend(metadataBackend, {
|
||||
limits: {
|
||||
@ -179,7 +182,8 @@ module.exports = class ApiRouter {
|
||||
statsBackend,
|
||||
layergroupMetadata,
|
||||
namedMapProviderCache,
|
||||
tablesExtentBackend
|
||||
tablesExtentBackend,
|
||||
clusterBackend
|
||||
};
|
||||
|
||||
this.mapRouter = new MapRouter({ collaborators });
|
||||
|
||||
@ -0,0 +1,91 @@
|
||||
'use strict';
|
||||
|
||||
const layergroupToken = require('../middlewares/layergroup-token');
|
||||
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
|
||||
const credentials = require('../middlewares/credentials');
|
||||
const dbConnSetup = require('../middlewares/db-conn-setup');
|
||||
const authorize = require('../middlewares/authorize');
|
||||
const rateLimit = require('../middlewares/rate-limit');
|
||||
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
|
||||
const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider');
|
||||
const cacheControlHeader = require('../middlewares/cache-control-header');
|
||||
const cacheChannelHeader = require('../middlewares/cache-channel-header');
|
||||
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
|
||||
const lastModifiedHeader = require('../middlewares/last-modified-header');
|
||||
|
||||
module.exports = class AggregatedFeaturesLayergroupController {
|
||||
constructor (
|
||||
clusterBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsBackend,
|
||||
layergroupAffectedTablesCache,
|
||||
authBackend,
|
||||
surrogateKeysCache
|
||||
) {
|
||||
this.clusterBackend = clusterBackend;
|
||||
this.pgConnection = pgConnection;
|
||||
this.mapStore = mapStore;
|
||||
this.userLimitsBackend = userLimitsBackend;
|
||||
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
|
||||
this.authBackend = authBackend;
|
||||
this.surrogateKeysCache = surrogateKeysCache;
|
||||
}
|
||||
|
||||
register (mapRouter) {
|
||||
mapRouter.get('/:token/:layer/cluster/:clusterId', this.middlewares());
|
||||
}
|
||||
|
||||
middlewares () {
|
||||
return [
|
||||
layergroupToken(),
|
||||
credentials(),
|
||||
authorize(this.authBackend),
|
||||
dbConnSetup(this.pgConnection),
|
||||
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
|
||||
cleanUpQueryParams(),
|
||||
createMapStoreMapConfigProvider(
|
||||
this.mapStore,
|
||||
this.userLimitsBackend,
|
||||
this.pgConnection,
|
||||
this.layergroupAffectedTablesCache
|
||||
),
|
||||
getClusteredFeatures(this.clusterBackend),
|
||||
cacheControlHeader(),
|
||||
cacheChannelHeader(),
|
||||
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
|
||||
lastModifiedHeader()
|
||||
];
|
||||
}
|
||||
};
|
||||
|
||||
function getClusteredFeatures (clusterBackend) {
|
||||
return function getFeatureAttributesMiddleware (req, res, next) {
|
||||
req.profiler.start('windshaft.maplayer_cluster_features');
|
||||
|
||||
const { mapConfigProvider } = res.locals;
|
||||
const { token } = res.locals;
|
||||
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
|
||||
const { layer, clusterId } = req.params;
|
||||
|
||||
const params = {
|
||||
token,
|
||||
dbuser, dbname, dbpassword, dbhost, dbport,
|
||||
layer, clusterId
|
||||
};
|
||||
|
||||
clusterBackend.getClusterFeatures(mapConfigProvider, params, (err, features, stats = {}) => {
|
||||
req.profiler.add(stats);
|
||||
|
||||
if (err) {
|
||||
err.label = 'GET CLUSTERED FEATURES';
|
||||
return next(err);
|
||||
}
|
||||
|
||||
res.statusCode = 200;
|
||||
res.body = features;
|
||||
|
||||
next();
|
||||
});
|
||||
};
|
||||
}
|
||||
@ -10,6 +10,7 @@ const TileLayergroupController = require('./tile-layergroup-controller');
|
||||
const AnonymousMapController = require('./anonymous-map-controller');
|
||||
const PreviewTemplateController = require('./preview-template-controller');
|
||||
const AnalysesCatalogController = require('./analyses-catalog-controller');
|
||||
const ClusteredFeaturesLayergroupController = require('./clustered-features-layergroup-controller');
|
||||
|
||||
module.exports = class MapRouter {
|
||||
constructor ({ collaborators }) {
|
||||
@ -32,7 +33,8 @@ module.exports = class MapRouter {
|
||||
statsBackend,
|
||||
layergroupMetadata,
|
||||
namedMapProviderCache,
|
||||
tablesExtentBackend
|
||||
tablesExtentBackend,
|
||||
clusterBackend
|
||||
} = collaborators;
|
||||
|
||||
this.analysisLayergroupController = new AnalysisLayergroupController(
|
||||
@ -112,6 +114,16 @@ module.exports = class MapRouter {
|
||||
authBackend,
|
||||
userLimitsBackend
|
||||
);
|
||||
|
||||
this.clusteredFeaturesLayergroupController = new ClusteredFeaturesLayergroupController(
|
||||
clusterBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsBackend,
|
||||
layergroupAffectedTablesCache,
|
||||
authBackend,
|
||||
surrogateKeysCache
|
||||
);
|
||||
}
|
||||
|
||||
register (apiRouter, mapPaths) {
|
||||
@ -125,6 +137,7 @@ module.exports = class MapRouter {
|
||||
this.anonymousMapController.register(mapRouter);
|
||||
this.previewTemplateController.register(mapRouter);
|
||||
this.analysesController.register(mapRouter);
|
||||
this.clusteredFeaturesLayergroupController.register(mapRouter);
|
||||
|
||||
mapPaths.forEach(path => apiRouter.use(path, mapRouter));
|
||||
}
|
||||
|
||||
168
lib/cartodb/backends/cluster.js
Normal file
168
lib/cartodb/backends/cluster.js
Normal file
@ -0,0 +1,168 @@
|
||||
'use strict';
|
||||
|
||||
const PSQL = require('cartodb-psql');
|
||||
const dbParamsFromReqParams = require('../utils/database-params');
|
||||
const debug = require('debug')('backend:cluster');
|
||||
|
||||
module.exports = class ClusterBackend {
|
||||
getClusterFeatures (mapConfigProvider, params, callback) {
|
||||
mapConfigProvider.getMapConfig((err, mapConfig) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
// if (!mapConfig.isAggregationLayer(params.layer)) {
|
||||
// const error = new Error(`Map ${params.token} has no aggregation defined for layer ${params.layer}`);
|
||||
// return callback(error);
|
||||
// }
|
||||
|
||||
const layer = mapConfig.getLayer(params.layer);
|
||||
|
||||
let pg;
|
||||
try {
|
||||
pg = new PSQL(dbParamsFromReqParams(params));
|
||||
} catch (error) {
|
||||
return callback(error);
|
||||
}
|
||||
|
||||
const query = layer.options.sql;
|
||||
const resolution = layer.options.aggregation.resolution || 1;
|
||||
|
||||
getColumnsName(pg, query, (err, columns) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
const { clusterId } = params;
|
||||
|
||||
getClusterFeatures(pg, clusterId, columns, query, resolution, (err, features) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
return callback(null, features);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const SKIP_COLUMNS = {
|
||||
'the_geom': true,
|
||||
'the_geom_webmercator': true
|
||||
};
|
||||
|
||||
function getColumnsName (pg, query, callback) {
|
||||
const sql = replaceTokens(limitedQuery({
|
||||
query: query
|
||||
}));
|
||||
|
||||
debug(sql);
|
||||
|
||||
pg.query(sql, function (err, resultSet) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
const fields = resultSet.fields || [];
|
||||
const columnNames = fields.map(field => field.name)
|
||||
.filter(columnName => !SKIP_COLUMNS[columnName]);
|
||||
|
||||
return callback(null, columnNames);
|
||||
}, true);
|
||||
}
|
||||
|
||||
function getClusterFeatures (pg, clusterId, columns, query, resolution, callback) {
|
||||
const sql = replaceTokens(clusterFeaturesQuery({
|
||||
id: clusterId,
|
||||
query: query,
|
||||
res: resolution,
|
||||
columns: columns
|
||||
}));
|
||||
|
||||
debug(sql);
|
||||
|
||||
pg.query(sql, (err, data) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
return callback(null, data);
|
||||
} , true); // use read-only transaction
|
||||
}
|
||||
|
||||
const SUBSTITUTION_TOKENS = {
|
||||
bbox: /!bbox!/g,
|
||||
scale_denominator: /!scale_denominator!/g,
|
||||
pixel_width: /!pixel_width!/g,
|
||||
pixel_height: /!pixel_height!/g,
|
||||
var_zoom: /@zoom/g,
|
||||
var_bbox: /@bbox/g,
|
||||
var_x: /@x/g,
|
||||
var_y: /@y/g,
|
||||
};
|
||||
|
||||
function replaceTokens(sql, replaceValues) {
|
||||
if (!sql) {
|
||||
return sql;
|
||||
}
|
||||
|
||||
replaceValues = replaceValues || {
|
||||
bbox: 'ST_MakeEnvelope(0,0,0,0)',
|
||||
scale_denominator: '0',
|
||||
pixel_width: '1',
|
||||
pixel_height: '1',
|
||||
var_zoom: '0',
|
||||
var_bbox: '[0,0,0,0]',
|
||||
var_x: '0',
|
||||
var_y: '0'
|
||||
};
|
||||
|
||||
Object.keys(replaceValues).forEach(function(token) {
|
||||
if (SUBSTITUTION_TOKENS[token]) {
|
||||
sql = sql.replace(SUBSTITUTION_TOKENS[token], replaceValues[token]);
|
||||
}
|
||||
});
|
||||
|
||||
return sql;
|
||||
}
|
||||
|
||||
const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
|
||||
// const nonGeomsQuery = ctx => `SELECT ${ctx.columns.join(', ')} FROM (${ctx.query}) __cdb_non_geoms_query`;
|
||||
const clusterFeaturesQuery = ctx => `
|
||||
WITH
|
||||
_cdb_params AS (
|
||||
SELECT
|
||||
${gridResolution(ctx)} AS res
|
||||
),
|
||||
_cell AS (
|
||||
SELECT
|
||||
ST_MakeEnvelope(
|
||||
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
|
||||
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
|
||||
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
|
||||
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
|
||||
3857
|
||||
) AS bbox
|
||||
FROM (${ctx.query}) _cdb_query, _cdb_params
|
||||
WHERE _cdb_query.cartodb_id = ${ctx.id}
|
||||
)
|
||||
SELECT ${ctx.columns.join(', ')} FROM (
|
||||
SELECT _cdb_query.* FROM _cell, (${ctx.query}) _cdb_query
|
||||
WHERE ST_Intersects(_cdb_query.the_geom_webmercator, _cell.bbox)
|
||||
) __cdb_non_geoms_query
|
||||
`;
|
||||
|
||||
// SQL expression to compute the aggregation resolution (grid cell size).
|
||||
// This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension
|
||||
// (i.e. each tile is divided into ctx.res*ctx.res cells).
|
||||
// We limit the the minimum resolution to avoid division by zero problems. The limit used is
|
||||
// the pixel size of zoom level 30 (i.e. 1/2*(30+8) of the full earth web-mercator extent), which is about 0.15 mm.
|
||||
// Computing this using !scale_denominator!, !pixel_width! or !pixel_height! produces
|
||||
// inaccurate results due to rounding present in those values.
|
||||
const gridResolution = ctx => {
|
||||
const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38);
|
||||
const pixelSize = 'CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))';
|
||||
debug(ctx);
|
||||
return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
|
||||
};
|
||||
49
test/acceptance/cluster.js
Normal file
49
test/acceptance/cluster.js
Normal file
@ -0,0 +1,49 @@
|
||||
'use strict';
|
||||
|
||||
require('../support/test_helper');
|
||||
|
||||
// const assert = require('../support/assert');
|
||||
const TestClient = require('../support/test-client');
|
||||
|
||||
const POINTS_SQL_1 = `
|
||||
select
|
||||
x + 4 as cartodb_id,
|
||||
st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom,
|
||||
st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator,
|
||||
x as value
|
||||
from generate_series(-3, 3) x
|
||||
`;
|
||||
|
||||
const defaultLayers = [{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: POINTS_SQL_1,
|
||||
aggregation: true
|
||||
}
|
||||
}];
|
||||
|
||||
function createVectorMapConfig (layers = defaultLayers) {
|
||||
return {
|
||||
version: '1.8.0',
|
||||
layers: layers
|
||||
};
|
||||
}
|
||||
|
||||
describe('cluster', function () {
|
||||
it.only('should get aggregated features of an aggregated map', function (done) {
|
||||
const mapConfig = createVectorMapConfig();
|
||||
const testClient = new TestClient(mapConfig);
|
||||
const clusterId = 1;
|
||||
const layerId = 0;
|
||||
const params = {};
|
||||
|
||||
testClient.getClusterFeatures(clusterId, layerId, params, (err, body) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
console.log('>>>>>>>>>>>>', body.rows);
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -620,6 +620,106 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params,
|
||||
);
|
||||
};
|
||||
|
||||
TestClient.prototype.getClusterFeatures = function(clusterId, layerId, params, callback) {
|
||||
var self = this;
|
||||
|
||||
if (!callback) {
|
||||
callback = params;
|
||||
params = {};
|
||||
}
|
||||
|
||||
var extraParams = {};
|
||||
|
||||
if (this.apiKey) {
|
||||
extraParams.api_key = this.apiKey;
|
||||
}
|
||||
|
||||
// if (params && params.filters) {
|
||||
// extraParams.filters = JSON.stringify(params.filters);
|
||||
// }
|
||||
|
||||
var url = '/api/v1/map';
|
||||
if (Object.keys(extraParams).length > 0) {
|
||||
url += '?' + qs.stringify(extraParams);
|
||||
}
|
||||
|
||||
var expectedResponse = params.response || {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8'
|
||||
}
|
||||
};
|
||||
|
||||
step(
|
||||
function createLayergroup() {
|
||||
var next = this;
|
||||
assert.response(self.server,
|
||||
{
|
||||
url: url,
|
||||
method: 'POST',
|
||||
headers: {
|
||||
host: 'localhost',
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
data: JSON.stringify(self.mapConfig)
|
||||
},
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8'
|
||||
}
|
||||
},
|
||||
function(res, err) {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
|
||||
var parsedBody = JSON.parse(res.body);
|
||||
|
||||
if (parsedBody.layergroupid) {
|
||||
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
|
||||
self.keysToDelete['user:localhost:mapviews:global'] = 5;
|
||||
}
|
||||
|
||||
return next(null, parsedBody.layergroupid);
|
||||
}
|
||||
);
|
||||
},
|
||||
function getCLusterFeatures(err, layergroupId) {
|
||||
assert.ifError(err);
|
||||
|
||||
var next = this;
|
||||
|
||||
url = '/api/v1/map/' + layergroupId + '/' + layerId + '/cluster/' + clusterId;
|
||||
|
||||
assert.response(self.server,
|
||||
{
|
||||
url: url,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost'
|
||||
}
|
||||
},
|
||||
expectedResponse,
|
||||
function(res, err) {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
|
||||
next(null, JSON.parse(res.body));
|
||||
}
|
||||
);
|
||||
},
|
||||
function finish(err, attributes) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
return callback(null, attributes);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
TestClient.prototype.getTile = function(z, x, y, params, callback) {
|
||||
var self = this;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user