From ab6004f21e8980e7d1130170c729ac2d3522979a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Mar 2016 14:34:57 +0100 Subject: [PATCH] Integrated turbo-cartocss for anonymous maps --- lib/cartodb/controllers/map.js | 41 ++++++++++---- lib/cartodb/server.js | 8 ++- .../utils/style/postgres-datasource.js | 55 +++++++++++++++++++ .../utils/style/turbo-cartocss-adapter.js | 55 +++++++++++++++++++ .../utils/style/turbo-cartocss-parser.js | 15 +++++ package.json | 3 +- test/acceptance/multilayer_server.js | 4 +- 7 files changed, 165 insertions(+), 16 deletions(-) create mode 100644 lib/cartodb/utils/style/postgres-datasource.js create mode 100644 lib/cartodb/utils/style/turbo-cartocss-adapter.js create mode 100644 lib/cartodb/utils/style/turbo-cartocss-parser.js diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ff164e9c..54d018f1 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -34,7 +34,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter') */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, overviewsMetadataApi, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables) { + surrogateKeysCache, userLimitsApi, layergroupAffectedTables, turboCartoCssAdapter) { BaseController.call(this, authApi, pgConnection); @@ -46,6 +46,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; + this.turboCartoCssAdapter = turboCartoCssAdapter; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi); @@ -152,21 +153,37 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { ); }, function addOverviewsInformation(err, requestMapConfig, datasource) { - assert.ifError(err); - var next = this; - self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, - function(err, layers) { - if (err) { - return next(err); - } + assert.ifError(err); + var next = this; + self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) { + if (err) { + return next(err); + } - if (layers) { - requestMapConfig.layers = layers; - } - return next(null, requestMapConfig, datasource); + if (layers) { + requestMapConfig.layers = layers; + } + + return next(null, requestMapConfig, datasource); } ); }, + function parseTurboCartoCss(err, requestMapConfig, datasource) { + assert.ifError(err); + + var next = this; + self.turboCartoCssAdapter.getLayers(req.context.user, requestMapConfig.layers, function (err, layers) { + if (err) { + return next(err); + } + + if (layers) { + requestMapConfig.layers = layers; + } + + return next(null, requestMapConfig, datasource); + }); + }, function createLayergroup(err, requestMapConfig, datasource) { assert.ifError(err); mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource()); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 39884850..ac1f185d 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -30,6 +30,8 @@ var PgConnection = require('./backends/pg_connection'); var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); +var TurboCartocssParser = require('./utils/style/turbo-cartocss-parser'); +var TurboCartocssAdapter = require('./utils/style/turbo-cartocss-adapter'); module.exports = function(serverOptions) { // Make stats client globally accessible @@ -150,6 +152,9 @@ module.exports = function(serverOptions) { var TablesExtentApi = require('./api/tables_extent_api'); var tablesExtentApi = new TablesExtentApi(pgQueryRunner); + var turboCartoCssParser = new TurboCartocssParser(pgQueryRunner); + var turboCartocssAdapter = new TurboCartocssAdapter(turboCartoCssParser); + /******************************************************************************************************************* * Routing ******************************************************************************************************************/ @@ -176,7 +181,8 @@ module.exports = function(serverOptions) { overviewsMetadataApi, surrogateKeysCache, userLimitsApi, - layergroupAffectedTablesCache + layergroupAffectedTablesCache, + turboCartocssAdapter ).register(app); new controller.NamedMaps( diff --git a/lib/cartodb/utils/style/postgres-datasource.js b/lib/cartodb/utils/style/postgres-datasource.js new file mode 100644 index 00000000..fd6e02c1 --- /dev/null +++ b/lib/cartodb/utils/style/postgres-datasource.js @@ -0,0 +1,55 @@ +'use strict'; + +var dot = require('dot'); +dot.templateSettings.strip = false; + +function createTemplate(method) { + return dot.template([ + 'SELECT', + method, + 'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL' + ].join('\n')); +} + +var methods = { + quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles', + equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal', + jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks', + headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails' +}; + +var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) { + methodTemplates[methodName] = createTemplate(methods[methodName]); + return methodTemplates; +}, {}); + +function PostgresDatasource (pgQueryRunner, username, query) { + this.pgQueryRunner = pgQueryRunner; + this.username = username; + this.query = query; +} + +PostgresDatasource.prototype.getName = function () { + return 'PostgresDatasource'; +}; + +PostgresDatasource.prototype.getRamp = function (column, buckets, method, callback) { + var methodName = methods.hasOwnProperty(method) ? method : 'quantiles'; + var template = methodTemplates[methodName]; + + var query = template({ _column: column, _sql: this.query, _buckets: buckets }); + + this.pgQueryRunner.run(this.username, query, function (err, result) { + if (err) { + return callback(err); + } + + var ramp = result[0][methodName].sort(function(a, b) { + return a - b; + }); + + return callback(null, ramp); + }); +}; + +module.exports = PostgresDatasource; diff --git a/lib/cartodb/utils/style/turbo-cartocss-adapter.js b/lib/cartodb/utils/style/turbo-cartocss-adapter.js new file mode 100644 index 00000000..873f8a2d --- /dev/null +++ b/lib/cartodb/utils/style/turbo-cartocss-adapter.js @@ -0,0 +1,55 @@ +'use strict'; + +var queue = require('queue-async'); + +function TurboCartocssAdapter(turboCartocssParser) { + this.turboCartocssParser = turboCartocssParser; +} + +module.exports = TurboCartocssAdapter; + +TurboCartocssAdapter.prototype.getLayers = function (username, layers, callback) { + var self = this; + + if (!layers || layers.length === 0) { + return callback(null, layers); + } + + var parseCartoCssQueue = queue(layers.length); + + layers.forEach(function(layer) { + parseCartoCssQueue.defer(self._parseCartoCss.bind(self), username, layer); + }); + + parseCartoCssQueue.awaitAll(function (err, layers) { + if (err) { + return callback(err); + } + + return callback(null, layers); + }); +}; + +TurboCartocssAdapter.prototype._parseCartoCss = function (username, layer, callback) { + if (isNotLayerToParseCartocss(layer)) { + return callback(null, layer); + } + + this.turboCartocssParser.process(username, layer.options.cartocss, layer.options.sql, function (err, cartocss) { + if (err) { + return callback(err); + } + + layer.options.cartocss = cartocss; + + callback(null, layer); + }); +}; + +function isNotLayerToParseCartocss(layer) { + if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' && layer.type !== 'torque' ) { + return true; + } + + return false; +} diff --git a/lib/cartodb/utils/style/turbo-cartocss-parser.js b/lib/cartodb/utils/style/turbo-cartocss-parser.js new file mode 100644 index 00000000..19a7f9c0 --- /dev/null +++ b/lib/cartodb/utils/style/turbo-cartocss-parser.js @@ -0,0 +1,15 @@ +'use strict'; + +var turboCartoCss = require('turbo-cartocss'); +var PostgresDatasource = require('./postgres-datasource'); + +function TurboCartocssParser (pgQueryRunner) { + this.pgQueryRunner = pgQueryRunner; +} + +module.exports = TurboCartocssParser; + +TurboCartocssParser.prototype.process = function (username, cartocss, sql, callback) { + var datasource = new PostgresDatasource(this.pgQueryRunner, username, sql); + turboCartoCss(cartocss, datasource, callback); +}; diff --git a/package.json b/package.json index 60eb890e..a225567c 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "lru-cache": "2.6.5", "lzma": "~1.3.7", "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb", - "cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" + "cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master", + "turbo-cartocss": "0.4.0" }, "devDependencies": { "istanbul": "~0.3.6", diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index e443f3aa..7462b0eb 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -71,8 +71,8 @@ describe('tests from old api translated to multilayer', function() { }, function(res) { var parsed = JSON.parse(res.body); - assert.ok(parsed.errors[0].match(/^style0/)); - assert.ok(parsed.errors[0].match(/missing closing/)); + assert.ok(parsed.errors[0]);q + assert.ok(parsed.errors[0].match(/Unclosed block/)); done(); } );