Integrated turbo-cartocss for anonymous maps

lib/cartodb/controllers/map.js

@@ -34,7 +34,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
  */
 function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
                        overviewsMetadataApi,
-                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, turboCartoCssAdapter) {
 
     BaseController.call(this, authApi, pgConnection);
 
@@ -46,6 +46,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
+    this.turboCartoCssAdapter = turboCartoCssAdapter;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
     this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi);
@@ -154,8 +155,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         function addOverviewsInformation(err, requestMapConfig, datasource) {
             assert.ifError(err);
             var next = this;
-          self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
+            self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) {
-              function(err, layers) {
                 if (err) {
                     return next(err);
                 }
@@ -163,10 +163,27 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
                 if (layers) {
                     requestMapConfig.layers = layers;
                 }
+
                 return next(null, requestMapConfig, datasource);
               }
           );
         },
+        function parseTurboCartoCss(err, requestMapConfig, datasource) {
+            assert.ifError(err);
+
+            var next = this;
+            self.turboCartoCssAdapter.getLayers(req.context.user, requestMapConfig.layers, function (err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    requestMapConfig.layers = layers;
+                }
+
+                return next(null, requestMapConfig, datasource);
+            });
+        },
         function createLayergroup(err, requestMapConfig, datasource) {
             assert.ifError(err);
             mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource());

lib/cartodb/server.js

@@ -30,6 +30,8 @@ var PgConnection = require('./backends/pg_connection');
 var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
 var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
+var TurboCartocssParser = require('./utils/style/turbo-cartocss-parser');
+var TurboCartocssAdapter = require('./utils/style/turbo-cartocss-adapter');
 
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
@@ -150,6 +152,9 @@ module.exports = function(serverOptions) {
     var TablesExtentApi = require('./api/tables_extent_api');
     var tablesExtentApi = new TablesExtentApi(pgQueryRunner);
 
+    var turboCartoCssParser = new TurboCartocssParser(pgQueryRunner);
+    var turboCartocssAdapter = new TurboCartocssAdapter(turboCartoCssParser);
+
     /*******************************************************************************************************************
      * Routing
      ******************************************************************************************************************/
@@ -176,7 +181,8 @@ module.exports = function(serverOptions) {
         overviewsMetadataApi,
         surrogateKeysCache,
         userLimitsApi,
-        layergroupAffectedTablesCache
+        layergroupAffectedTablesCache,
+        turboCartocssAdapter
     ).register(app);
 
     new controller.NamedMaps(

lib/cartodb/utils/style/postgres-datasource.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function createTemplate(method) {
+    return dot.template([
+        'SELECT',
+        method,
+        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
+    ].join('\n'));
+}
+
+var methods = {
+    quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
+    equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
+    jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
+};
+
+var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {
+    methodTemplates[methodName] = createTemplate(methods[methodName]);
+    return methodTemplates;
+}, {});
+
+function PostgresDatasource (pgQueryRunner, username, query) {
+    this.pgQueryRunner = pgQueryRunner;
+    this.username = username;
+    this.query = query;
+}
+
+PostgresDatasource.prototype.getName = function () {
+    return 'PostgresDatasource';
+};
+
+PostgresDatasource.prototype.getRamp = function (column, buckets, method, callback) {
+    var methodName = methods.hasOwnProperty(method) ? method : 'quantiles';
+    var template = methodTemplates[methodName];
+
+    var query = template({ _column: column, _sql: this.query, _buckets: buckets });
+
+    this.pgQueryRunner.run(this.username, query, function (err, result) {
+        if (err) {
+            return callback(err);
+        }
+
+        var ramp = result[0][methodName].sort(function(a, b) {
+            return a - b;
+        });
+
+        return callback(null, ramp);
+    });
+};
+
+module.exports = PostgresDatasource;

lib/cartodb/utils/style/turbo-cartocss-adapter.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var queue = require('queue-async');
+
+function TurboCartocssAdapter(turboCartocssParser) {
+    this.turboCartocssParser = turboCartocssParser;
+}
+
+module.exports = TurboCartocssAdapter;
+
+TurboCartocssAdapter.prototype.getLayers = function (username, layers, callback) {
+    var self = this;
+
+    if (!layers || layers.length === 0) {
+        return callback(null, layers);
+    }
+
+    var parseCartoCssQueue = queue(layers.length);
+
+    layers.forEach(function(layer) {
+        parseCartoCssQueue.defer(self._parseCartoCss.bind(self), username, layer);
+    });
+
+    parseCartoCssQueue.awaitAll(function (err, layers) {
+        if (err) {
+            return callback(err);
+        }
+
+        return callback(null, layers);
+    });
+};
+
+TurboCartocssAdapter.prototype._parseCartoCss = function (username, layer, callback) {
+    if (isNotLayerToParseCartocss(layer)) {
+        return callback(null, layer);
+    }
+
+    this.turboCartocssParser.process(username, layer.options.cartocss, layer.options.sql, function (err, cartocss) {
+        if (err) {
+            return callback(err);
+        }
+
+        layer.options.cartocss = cartocss;
+
+        callback(null, layer);
+    });
+};
+
+function isNotLayerToParseCartocss(layer) {
+    if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' && layer.type !== 'torque' ) {
+        return true;
+    }
+
+    return false;
+}

lib/cartodb/utils/style/turbo-cartocss-parser.js

@@ -0,0 +1,15 @@
+'use strict';
+
+var turboCartoCss = require('turbo-cartocss');
+var PostgresDatasource = require('./postgres-datasource');
+
+function TurboCartocssParser (pgQueryRunner) {
+    this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = TurboCartocssParser;
+
+TurboCartocssParser.prototype.process = function (username, cartocss, sql, callback) {
+    var datasource = new PostgresDatasource(this.pgQueryRunner, username, sql);
+    turboCartoCss(cartocss, datasource, callback);
+};

package.json

@@ -37,7 +37,8 @@
         "lru-cache": "2.6.5",
         "lzma": "~1.3.7",
         "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
-        "cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
+        "cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master",
+        "turbo-cartocss": "0.4.0"
     },
     "devDependencies": {
         "istanbul": "~0.3.6",

test/acceptance/multilayer_server.js

@@ -71,8 +71,8 @@ describe('tests from old api translated to multilayer', function() {
             },
             function(res) {
                 var parsed = JSON.parse(res.body);
-                assert.ok(parsed.errors[0].match(/^style0/));
+                assert.ok(parsed.errors[0]);q
-                assert.ok(parsed.errors[0].match(/missing closing/));
+                assert.ok(parsed.errors[0].match(/Unclosed block/));
                 done();
             }
         );