From 354c982ea03d45d4fd6fedbfa59686db3801304b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 12 Feb 2016 16:12:02 +0100 Subject: [PATCH 01/80] Fix jsdoc --- lib/cartodb/controllers/map.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ee49e0eb..a5555a2f 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -27,6 +27,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter') * @param {MapBackend} mapBackend * @param metadataBackend * @param {QueryTablesApi} queryTablesApi + * @param {OverviewsMetadataApi} overviewsMetadataApi * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables From bcf3ce71ef4190318fee4dba7702090c2c97b345 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 12 Feb 2016 18:38:06 +0100 Subject: [PATCH 02/80] Adds experimental adapter to use queries based on camshaft analysis --- lib/cartodb/controllers/map.js | 6 + .../mapconfig_analysis_layers_adapter.js | 128 ++++++++++++++++++ package.json | 1 + 3 files changed, 135 insertions(+) create mode 100644 lib/cartodb/models/mapconfig_analysis_layers_adapter.js diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index a5555a2f..a09b9572 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -16,6 +16,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var TablesCacheEntry = require('../cache/model/database_tables_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); +var MapConfigAnalysisLayersAdapter = require('../models/mapconfig_analysis_layers_adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter'); @@ -49,6 +50,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; + this.analysisLayersAdapter = new MapConfigAnalysisLayersAdapter(); this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi); } @@ -137,6 +139,10 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { self.req2params(req, this); }, prepareConfigFn, + function prepareAnalysisLayers(err, requestMapConfig) { + assert.ifError(err); + self.analysisLayersAdapter.getLayers(req.context.user, requestMapConfig, this); + }, function beforeLayergroupCreate(err, requestMapConfig) { assert.ifError(err); var next = this; diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js new file mode 100644 index 00000000..920dfa97 --- /dev/null +++ b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js @@ -0,0 +1,128 @@ +var queue = require('queue-async'); +var _ = require('underscore'); +var Datasource = require('windshaft').model.Datasource; + +var camshaft = require('camshaft'); +var dot = require('dot'); +dot.templateSettings.strip = false; + +function MapConfigAnalysisLayersAdapter(templateMaps) { + this.templateMaps = templateMaps; +} + +module.exports = MapConfigAnalysisLayersAdapter; + + +multitypeStyleTemplate = dot.template([ + "#points['mapnik::geometry_type'=1] {", + " marker-fill-opacity: {{=it._opacity}};", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: {{=it._opacity}};", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 4;", + " marker-fill: {{=it._color}};", + " marker-allow-overlap: true;", + "}", + "#lines['mapnik::geometry_type'=2] {", + " line-color: {{=it._color}};", + " line-width: 2;", + " line-opacity: {{=it._opacity}};", + "}", + "#polygons['mapnik::geometry_type'=3] {", + " polygon-fill: {{=it._color}};", + " polygon-opacity: 0.7;", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: {{=it._opacity}};", + "}" +].join('\n')); + +function multiTypeStyle(color, opacity) { + return multitypeStyleTemplate({ + _opacity: opacity || 1.0, + _color: color || 'red' + }); +} + +var layerQueryTemplate = dot.template([ + 'SELECT ST_Transform(the_geom, 3857) the_geom_webmercator', + 'FROM ({{=it._query}}) _cdb_analysis_query' +].join('\n')); + +function getLayer(query, color, opacity) { + + console.log(multiTypeStyle(color, opacity)); + return { + type: 'mapnik', + options: { + sql: layerQueryTemplate({ _query: query } ), + cartocss: multiTypeStyle(color, opacity), + cartocss_version: '2.3.0' + } + }; +} + +MapConfigAnalysisLayersAdapter.prototype.getLayers = function(username, requestMapConfig, callback) { + + function adaptLayer(layer, done) { + if (isAnalysisTypeLayer(layer)) { + + var analysisDefinition = JSON.parse(layer.options.def); + console.log(JSON.stringify(analysisDefinition, null, 2)); + + camshaft.create(username, analysisDefinition, function(err, analysis) { + if (err) { + return done(err); + } + + var layers = []; + + analysis.getSortedNodes().reverse().forEach(function(node) { + layers.push(getLayer(node.getQuery(), 'grey', 0.2)); + }); + + layers.push(getLayer(analysis.getQuery())); + + return done(null, { layers: layers }); + }); + } else { + return done(null, { layers: [layer] }); + } + } + + function layersAdaptQueueFinish(err, layersResults) { + if (err) { + return callback(err); + } + + if (!layersResults || layersResults.length === 0) { + return callback(new Error('Missing layers array from layergroup config')); + } + + var layers = []; + layersResults.forEach(function(layersResult) { + layers = layers.concat(layersResult.layers); + }); + + requestMapConfig.layers = layers; + + return callback(null, requestMapConfig); + } + + var adaptLayersQueue = queue(requestMapConfig.layers.length); + + if (_.some(requestMapConfig.layers, isAnalysisTypeLayer)) { + requestMapConfig.layers.forEach(function(layer) { + adaptLayersQueue.defer(adaptLayer, layer); + }); + adaptLayersQueue.awaitAll(layersAdaptQueueFinish); + } else { + return callback(null, requestMapConfig); + } +}; + +function isAnalysisTypeLayer(layer) { + return layer.type === 'analysis'; +} diff --git a/package.json b/package.json index e31e26ff..4f26468b 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "underscore" : "~1.6.0", "dot": "~1.0.2", "windshaft": "1.9.0", + "camshaft": "https://github.com/CartoDB/camshaft/tarball/master", "step": "~0.0.6", "queue-async": "~1.0.7", "request": "~2.62.0", From ed1f753690ce9a3714086c4fee87903b01d17137 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 12 Feb 2016 18:45:46 +0100 Subject: [PATCH 03/80] Fix style --- lib/cartodb/models/mapconfig_analysis_layers_adapter.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js index 920dfa97..54bfe6a3 100644 --- a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js +++ b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js @@ -1,6 +1,5 @@ var queue = require('queue-async'); var _ = require('underscore'); -var Datasource = require('windshaft').model.Datasource; var camshaft = require('camshaft'); var dot = require('dot'); @@ -13,7 +12,7 @@ function MapConfigAnalysisLayersAdapter(templateMaps) { module.exports = MapConfigAnalysisLayersAdapter; -multitypeStyleTemplate = dot.template([ +var multitypeStyleTemplate = dot.template([ "#points['mapnik::geometry_type'=1] {", " marker-fill-opacity: {{=it._opacity}};", " marker-line-color: #FFF;", From 2f51ad9c3fe31e98a9bba030f3ef2b58535e702b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 12 Feb 2016 18:49:01 +0100 Subject: [PATCH 04/80] Regenerate npm-shrinkwrap.json to include camshaft dep --- npm-shrinkwrap.json | 407 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 405 insertions(+), 2 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 70e7c50e..81f0b632 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -99,6 +99,408 @@ } } }, + "camshaft": { + "version": "0.1.0", + "from": "https://github.com/CartoDB/camshaft/tarball/master", + "resolved": "https://github.com/CartoDB/camshaft/tarball/master", + "dependencies": { + "async": { + "version": "1.5.2", + "from": "async@>=1.5.2 <2.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" + }, + "cartodb-psql": { + "version": "0.6.1", + "from": "cartodb-psql@>=0.6.1 <0.7.0", + "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.6.1.tgz", + "dependencies": { + "pg": { + "version": "2.6.2-cdb3", + "from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb3", + "resolved": "git://github.com/CartoDB/node-postgres.git#069c5296d1a093077feff21719641bb9e71fc50e", + "dependencies": { + "generic-pool": { + "version": "2.0.3", + "from": "generic-pool@2.0.3", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.0.3.tgz" + }, + "buffer-writer": { + "version": "1.0.0", + "from": "buffer-writer@1.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz" + } + } + } + } + }, + "request": { + "version": "2.69.0", + "from": "request@>=2.69.0 <3.0.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.69.0.tgz", + "dependencies": { + "aws-sign2": { + "version": "0.6.0", + "from": "aws-sign2@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz" + }, + "aws4": { + "version": "1.2.1", + "from": "aws4@>=1.2.1 <2.0.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.2.1.tgz" + }, + "bl": { + "version": "1.0.3", + "from": "bl@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz", + "dependencies": { + "readable-stream": { + "version": "2.0.5", + "from": "readable-stream@>=2.0.5 <2.1.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.5.tgz", + "dependencies": { + "core-util-is": { + "version": "1.0.2", + "from": "core-util-is@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "inherits": { + "version": "2.0.1", + "from": "inherits@>=2.0.1 <2.1.0", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + }, + "isarray": { + "version": "0.0.1", + "from": "isarray@0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + }, + "process-nextick-args": { + "version": "1.0.6", + "from": "process-nextick-args@>=1.0.6 <1.1.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "string_decoder@>=0.10.0 <0.11.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + }, + "util-deprecate": { + "version": "1.0.2", + "from": "util-deprecate@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + } + } + } + } + }, + "caseless": { + "version": "0.11.0", + "from": "caseless@>=0.11.0 <0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz" + }, + "combined-stream": { + "version": "1.0.5", + "from": "combined-stream@>=1.0.5 <1.1.0", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", + "dependencies": { + "delayed-stream": { + "version": "1.0.0", + "from": "delayed-stream@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + } + } + }, + "extend": { + "version": "3.0.0", + "from": "extend@>=3.0.0 <3.1.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" + }, + "forever-agent": { + "version": "0.6.1", + "from": "forever-agent@>=0.6.1 <0.7.0", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" + }, + "form-data": { + "version": "1.0.0-rc3", + "from": "form-data@>=1.0.0-rc3 <1.1.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc3.tgz" + }, + "har-validator": { + "version": "2.0.6", + "from": "har-validator@>=2.0.6 <2.1.0", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", + "dependencies": { + "chalk": { + "version": "1.1.1", + "from": "chalk@>=1.1.1 <2.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz", + "dependencies": { + "ansi-styles": { + "version": "2.1.0", + "from": "ansi-styles@>=2.1.0 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz" + }, + "escape-string-regexp": { + "version": "1.0.4", + "from": "escape-string-regexp@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.4.tgz" + }, + "has-ansi": { + "version": "2.0.0", + "from": "has-ansi@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "ansi-regex@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + }, + "strip-ansi": { + "version": "3.0.0", + "from": "strip-ansi@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "ansi-regex@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + }, + "supports-color": { + "version": "2.0.0", + "from": "supports-color@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" + } + } + }, + "commander": { + "version": "2.9.0", + "from": "commander@>=2.9.0 <3.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", + "dependencies": { + "graceful-readlink": { + "version": "1.0.1", + "from": "graceful-readlink@>=1.0.0", + "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz" + } + } + }, + "is-my-json-valid": { + "version": "2.12.4", + "from": "is-my-json-valid@>=2.12.4 <3.0.0", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.4.tgz", + "dependencies": { + "generate-function": { + "version": "2.0.0", + "from": "generate-function@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz" + }, + "generate-object-property": { + "version": "1.2.0", + "from": "generate-object-property@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "dependencies": { + "is-property": { + "version": "1.0.2", + "from": "is-property@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz" + } + } + }, + "jsonpointer": { + "version": "2.0.0", + "from": "jsonpointer@2.0.0", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz" + }, + "xtend": { + "version": "4.0.1", + "from": "xtend@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + } + } + }, + "pinkie-promise": { + "version": "2.0.0", + "from": "pinkie-promise@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.0.tgz", + "dependencies": { + "pinkie": { + "version": "2.0.4", + "from": "pinkie@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" + } + } + } + } + }, + "hawk": { + "version": "3.1.3", + "from": "hawk@>=3.1.0 <3.2.0", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", + "dependencies": { + "hoek": { + "version": "2.16.3", + "from": "hoek@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz" + }, + "boom": { + "version": "2.10.1", + "from": "boom@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz" + }, + "cryptiles": { + "version": "2.0.5", + "from": "cryptiles@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz" + }, + "sntp": { + "version": "1.0.9", + "from": "sntp@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz" + } + } + }, + "http-signature": { + "version": "1.1.1", + "from": "http-signature@>=1.1.0 <1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", + "dependencies": { + "assert-plus": { + "version": "0.2.0", + "from": "assert-plus@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz" + }, + "jsprim": { + "version": "1.2.2", + "from": "jsprim@>=1.2.2 <2.0.0", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.2.2.tgz", + "dependencies": { + "extsprintf": { + "version": "1.0.2", + "from": "extsprintf@1.0.2", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz" + }, + "json-schema": { + "version": "0.2.2", + "from": "json-schema@0.2.2", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz" + }, + "verror": { + "version": "1.3.6", + "from": "verror@1.3.6", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz" + } + } + }, + "sshpk": { + "version": "1.7.4", + "from": "sshpk@>=1.7.0 <2.0.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.7.4.tgz", + "dependencies": { + "asn1": { + "version": "0.2.3", + "from": "asn1@>=0.2.3 <0.3.0", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz" + }, + "dashdash": { + "version": "1.13.0", + "from": "dashdash@>=1.10.1 <2.0.0", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.13.0.tgz", + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } + }, + "jsbn": { + "version": "0.1.0", + "from": "jsbn@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz" + }, + "tweetnacl": { + "version": "0.13.3", + "from": "tweetnacl@>=0.13.0 <1.0.0", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.13.3.tgz" + }, + "jodid25519": { + "version": "1.0.2", + "from": "jodid25519@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz" + }, + "ecc-jsbn": { + "version": "0.1.1", + "from": "ecc-jsbn@>=0.0.1 <1.0.0", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz" + } + } + } + } + }, + "is-typedarray": { + "version": "1.0.0", + "from": "is-typedarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" + }, + "isstream": { + "version": "0.1.2", + "from": "isstream@>=0.1.2 <0.2.0", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" + }, + "json-stringify-safe": { + "version": "5.0.1", + "from": "json-stringify-safe@>=5.0.1 <5.1.0", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + }, + "mime-types": { + "version": "2.1.9", + "from": "mime-types@>=2.1.7 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "dependencies": { + "mime-db": { + "version": "1.21.0", + "from": "mime-db@>=1.21.0 <1.22.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + } + } + }, + "node-uuid": { + "version": "1.4.7", + "from": "node-uuid@>=1.4.7 <1.5.0", + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz" + }, + "oauth-sign": { + "version": "0.8.1", + "from": "oauth-sign@>=0.8.0 <0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.1.tgz" + }, + "qs": { + "version": "6.0.2", + "from": "qs@>=6.0.2 <6.1.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.0.2.tgz" + }, + "stringstream": { + "version": "0.0.5", + "from": "stringstream@>=0.0.4 <0.1.0", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz" + }, + "tough-cookie": { + "version": "2.2.1", + "from": "tough-cookie@>=2.2.0 <2.3.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.1.tgz" + }, + "tunnel-agent": { + "version": "0.4.2", + "from": "tunnel-agent@>=0.4.1 <0.5.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.2.tgz" + } + } + } + } + }, "cartodb-psql": { "version": "0.4.0", "from": "cartodb-psql@>=0.4.0 <0.5.0", @@ -520,9 +922,9 @@ "resolved": "https://registry.npmjs.org/request/-/request-2.62.0.tgz", "dependencies": { "bl": { - "version": "1.0.2", + "version": "1.0.3", "from": "bl@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz", "dependencies": { "readable-stream": { "version": "2.0.5", @@ -832,6 +1234,7 @@ "windshaft": { "version": "1.9.0", "from": "windshaft@1.9.0", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.9.0.tgz", "dependencies": { "mapnik": { "version": "1.4.15-cdb6", From 87b4a37f3f571501c33aeae15c442c4b4d472d78 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 17 Feb 2016 11:29:40 +0100 Subject: [PATCH 05/80] Move camshaft dep to avoid future conflicts while updating windshaft --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cd00f8fa..9915f380 100644 --- a/package.json +++ b/package.json @@ -27,12 +27,12 @@ "underscore" : "~1.6.0", "dot": "~1.0.2", "windshaft": "1.10.1", - "camshaft": "https://github.com/CartoDB/camshaft/tarball/master", "step": "~0.0.6", "queue-async": "~1.0.7", "request": "~2.62.0", "cartodb-redis": "~0.13.0", "cartodb-psql": "~0.4.0", + "camshaft": "https://github.com/CartoDB/camshaft/tarball/master", "fastly-purge": "~1.0.1", "redis-mpool": "~0.4.0", "lru-cache": "2.6.5", From 8d4ebc171b2a5cbb6cede04fa949fd4475275ecd Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 17 Feb 2016 12:15:20 +0100 Subject: [PATCH 06/80] Use a set to compare surrogate keys, avoiding key order errors --- test/support/test_helper.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/support/test_helper.js b/test/support/test_helper.js index e44916fd..de0f7822 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -63,7 +63,15 @@ function checkCache(res) { function checkSurrogateKey(res, expectedKey) { assert.ok(res.headers.hasOwnProperty('surrogate-key')); - assert.equal(res.headers['surrogate-key'], expectedKey); + + function createSet(keys, key) { + keys[key] = true; + return keys; + } + var keys = res.headers['surrogate-key'].split(' ').reduce(createSet, {}); + var expectedKeys = expectedKey.split(' ').reduce(createSet, {}); + + assert.deepEqual(keys, expectedKeys); } //global afterEach to capture test suites that leave keys in redis From 6d911726301747e5636d0a304260bcb10d4453f5 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 18 Feb 2016 13:43:39 +0100 Subject: [PATCH 07/80] Remove console.log --- lib/cartodb/models/mapconfig_analysis_layers_adapter.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js index 54bfe6a3..54cc8ac6 100644 --- a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js +++ b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js @@ -51,8 +51,6 @@ var layerQueryTemplate = dot.template([ ].join('\n')); function getLayer(query, color, opacity) { - - console.log(multiTypeStyle(color, opacity)); return { type: 'mapnik', options: { @@ -69,7 +67,6 @@ MapConfigAnalysisLayersAdapter.prototype.getLayers = function(username, requestM if (isAnalysisTypeLayer(layer)) { var analysisDefinition = JSON.parse(layer.options.def); - console.log(JSON.stringify(analysisDefinition, null, 2)); camshaft.create(username, analysisDefinition, function(err, analysis) { if (err) { From 30f8234bd0f4027864c450417b823835656b6ca1 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 19 Feb 2016 17:13:28 +0100 Subject: [PATCH 08/80] Use analysis configuration as per new camshaft api --- lib/cartodb/controllers/map.js | 17 ++++++++++++++++- .../models/mapconfig_analysis_layers_adapter.js | 8 ++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 1d05a120..f5c8b39b 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -141,7 +141,22 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { prepareConfigFn, function prepareAnalysisLayers(err, requestMapConfig) { assert.ifError(err); - self.analysisLayersAdapter.getLayers(req.context.user, requestMapConfig, this); + var analysisConfiguration = { + db: { + host: req.params.dbhost, + port: req.params.dbport, + dbname: req.params.dbname, + user: req.params.dbuser, + pass: req.params.dbpassword + }, + batch: { + // TODO load this from configuration + endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', + username: req.context.user, + apiKey: req.params.api_key + } + }; + self.analysisLayersAdapter.getLayers(analysisConfiguration, requestMapConfig, this); }, function beforeLayergroupCreate(err, requestMapConfig) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js index 54cc8ac6..703216f4 100644 --- a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js +++ b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js @@ -61,14 +61,18 @@ function getLayer(query, color, opacity) { }; } -MapConfigAnalysisLayersAdapter.prototype.getLayers = function(username, requestMapConfig, callback) { +MapConfigAnalysisLayersAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, callback) { + + if (!Array.isArray(requestMapConfig.layers)) { + return callback(null, requestMapConfig); + } function adaptLayer(layer, done) { if (isAnalysisTypeLayer(layer)) { var analysisDefinition = JSON.parse(layer.options.def); - camshaft.create(username, analysisDefinition, function(err, analysis) { + camshaft.create(analysisConfiguration, analysisDefinition, function(err, analysis) { if (err) { return done(err); } From a44477dddc1310ac175c4efa74ad5dea4d3260c7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 2 Mar 2016 12:40:53 +0100 Subject: [PATCH 09/80] TestClient with method to retrieve tiles --- test/support/test-client.js | 113 +++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index d2df611b..1a29f564 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -3,6 +3,8 @@ var qs = require('querystring'); var step = require('step'); +var mapnik = require('windshaft').mapnik; + var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); var assert = require('./assert'); @@ -13,8 +15,9 @@ var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); -function TestClient(mapConfig) { +function TestClient(mapConfig, apiKey) { this.mapConfig = mapConfig; + this.apiKey = apiKey; this.keysToDelete = {}; } @@ -114,6 +117,114 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { ); }; +TestClient.prototype.getTile = function(z, x, y, params, callback) { + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + var url = '/api/v1/map'; + + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + var layergroupId; + step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).layergroupid); + } + ); + }, + function getTileResult(err, _layergroupId) { + assert.ifError(err); + + var next = this; + layergroupId = _layergroupId; + + url = '/api/v1/map/' + layergroupId + '/'; + + var layers = params.layers; + if (!!layers) { + layers = Array.isArray(layers) ? layers : [layers]; + url += layers.join(',') + '/'; + } + + var format = params.format || 'png'; + + url += [z,x,y].join('/'); + url += '.' + format; + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }; + + + var expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + var isPng = format === 'png'; + + if (isPng) { + request.encoding = 'binary'; + expectedResponse.headers['Content-Type'] = 'image/png'; + } + + assert.response(server, request, expectedResponse, function(res, err) { + assert.ifError(err); + + var image; + + if (isPng) { + image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + } + + next(null, res, image); + }); + }, + function finish(err, res, image) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + return callback(err, res, image); + } + ); +}; + TestClient.prototype.drain = function(callback) { helper.deleteRedisKeys(this.keysToDelete, callback); }; From ce032fcc969aa49e097d9f4a074e68ff64cd88e3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 2 Mar 2016 12:42:42 +0100 Subject: [PATCH 10/80] Improve styling in analysis layers --- .../mapconfig_analysis_layers_adapter.js | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js index 703216f4..2b000ecc 100644 --- a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js +++ b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js @@ -11,6 +11,15 @@ function MapConfigAnalysisLayersAdapter(templateMaps) { module.exports = MapConfigAnalysisLayersAdapter; +var SKIP_COLUMNS = { + 'the_geom': true, + 'the_geom_webmercator': true +}; + +function skipColumns(columnNames) { + return columnNames + .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; }); +} var multitypeStyleTemplate = dot.template([ "#points['mapnik::geometry_type'=1] {", @@ -31,7 +40,7 @@ var multitypeStyleTemplate = dot.template([ "}", "#polygons['mapnik::geometry_type'=3] {", " polygon-fill: {{=it._color}};", - " polygon-opacity: 0.7;", + " polygon-opacity: {{=it._opacity}};", " line-color: #FFF;", " line-width: 0.5;", " line-opacity: {{=it._opacity}};", @@ -46,21 +55,36 @@ function multiTypeStyle(color, opacity) { } var layerQueryTemplate = dot.template([ - 'SELECT ST_Transform(the_geom, 3857) the_geom_webmercator', + 'SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, {{=it._columns}}', 'FROM ({{=it._query}}) _cdb_analysis_query' ].join('\n')); -function getLayer(query, color, opacity) { +function layerQuery(query, columnNames) { + return layerQueryTemplate({ _query: query, _columns: skipColumns(columnNames).join(', ') }); +} + +function getLayer(query, columnNames, color, opacity) { return { type: 'mapnik', options: { - sql: layerQueryTemplate({ _query: query } ), + sql: layerQuery(query, columnNames), cartocss: multiTypeStyle(color, opacity), cartocss_version: '2.3.0' } }; } +function getLayerWithStyle(query, columnNames, cartocss) { + return { + type: 'mapnik', + options: { + sql: layerQuery(query, columnNames), + cartocss: cartocss, + cartocss_version: '2.3.0' + } + }; +} + MapConfigAnalysisLayersAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, callback) { if (!Array.isArray(requestMapConfig.layers)) { @@ -79,11 +103,19 @@ MapConfigAnalysisLayersAdapter.prototype.getLayers = function(analysisConfigurat var layers = []; - analysis.getSortedNodes().reverse().forEach(function(node) { - layers.push(getLayer(node.getQuery(), 'grey', 0.2)); + analysis.getSortedNodes().reverse().forEach(function(node, i) { + var layer = getLayer(node.getQuery(), node.getColumns(), '#333', 0.1 * i); + if (!!node.params && !!node.params.cartocss) { + layer = getLayerWithStyle(node.getQuery(), node.getColumns(), node.params.cartocss); + } + layers.push(layer); }); - layers.push(getLayer(analysis.getQuery())); + layers.push(getLayerWithStyle( + analysis.getQuery(), + analysis.getRoot().getColumns(), + layer.options.cartocss + )); return done(null, { layers: layers }); }); From 6823fd8b03402919bf89c61d32e46034700abf4a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 2 Mar 2016 12:43:00 +0100 Subject: [PATCH 11/80] Better datasets for analysis --- test/support/sql/windshaft.test.sql | 374 ++++++++++++++++++++++++++++ 1 file changed, 374 insertions(+) diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 8a01903b..cb8485b1 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -332,3 +332,377 @@ INSERT INTO _vovw_2_test_table_overviews VALUES INSERT INTO _vovw_1_test_table_overviews VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'); + + +-- analysis tables ----------------------------------------------- + + + +drop table if exists cdb_analysis_catalog; +create table cdb_analysis_catalog ( + node_id char(40) CONSTRAINT cdb_analysis_catalog_pkey PRIMARY KEY, + analysis_def json NOT NULL, + input_nodes char(40) ARRAY NOT NULL DEFAULT '{}', + affected_tables regclass[] NOT NULL DEFAULT '{}', + cache_tables regclass[] NOT NULL DEFAULT '{}', + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now(), + used_at timestamp with time zone NOT NULL DEFAULT now(), + hits NUMERIC DEFAULT 0, + last_used_from char(40) +); + +ALTER TABLE cdb_analysis_catalog OWNER TO :TESTUSER; + + +-- +-- TOC entry 804 (class 1259 OID 13870252) +-- Name: analysis_banks; Type: TABLE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE TABLE analysis_banks ( + cartodb_id bigint NOT NULL, + the_geom geometry(Geometry,4326), + the_geom_webmercator geometry(Geometry,3857), + bank text +); + + +ALTER TABLE analysis_banks OWNER TO :TESTUSER; + +-- +-- TOC entry 803 (class 1259 OID 13870250) +-- Name: analysis_banks_cartodb_id_seq; Type: SEQUENCE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE SEQUENCE analysis_banks_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE analysis_banks_cartodb_id_seq OWNER TO :TESTUSER; + +-- +-- TOC entry 5784 (class 0 OID 0) +-- Dependencies: 803 +-- Name: analysis_banks_cartodb_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER SEQUENCE analysis_banks_cartodb_id_seq OWNED BY analysis_banks.cartodb_id; + + +-- +-- TOC entry 802 (class 1259 OID 13870235) +-- Name: analysis_rent_listings; Type: TABLE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE TABLE analysis_rent_listings ( + cartodb_id bigint NOT NULL, + the_geom geometry(Geometry,4326), + the_geom_webmercator geometry(Geometry,3857), + price double precision +); + + +ALTER TABLE analysis_rent_listings OWNER TO :TESTUSER; + +-- +-- TOC entry 801 (class 1259 OID 13870233) +-- Name: analysis_rent_listings_cartodb_id_seq; Type: SEQUENCE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE SEQUENCE analysis_rent_listings_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE analysis_rent_listings_cartodb_id_seq OWNER TO :TESTUSER; + +-- +-- TOC entry 5786 (class 0 OID 0) +-- Dependencies: 801 +-- Name: analysis_rent_listings_cartodb_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER SEQUENCE analysis_rent_listings_cartodb_id_seq OWNED BY analysis_rent_listings.cartodb_id; + + +-- +-- TOC entry 5612 (class 2604 OID 13870258) +-- Name: cartodb_id; Type: DEFAULT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER TABLE ONLY analysis_banks ALTER COLUMN cartodb_id SET DEFAULT nextval('analysis_banks_cartodb_id_seq'::regclass); + + +-- +-- TOC entry 5611 (class 2604 OID 13870241) +-- Name: cartodb_id; Type: DEFAULT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER TABLE ONLY analysis_rent_listings ALTER COLUMN cartodb_id SET DEFAULT nextval('analysis_rent_listings_cartodb_id_seq'::regclass); + + +-- +-- TOC entry 5778 (class 0 OID 13870252) +-- Dependencies: 804 +-- Data for Name: analysis_banks; Type: TABLE DATA; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +COPY analysis_banks (cartodb_id, the_geom, the_geom_webmercator, bank) FROM stdin; +1 0101000020E61000000AEC3F6BE5A80DC09EF022E20C364440 0101000020110F0000AD24852BA63019C13215440E02CC5241 BBVA +2 0101000020E61000005AB3C72EE6A40DC02A499181E3364440 0101000020110F0000DE3E9A22412D19C1B059CF80F1CC5241 Santander +3 0101000020E6100000FD52FCA1E1960DC0DD48F7ADB9354440 0101000020110F00008C1CE860592119C12B37FC3BA5CB5241 BBVA +4 0101000020E61000008A52C823B69F0DC02579E6E0B4354440 0101000020110F0000DC41553AD92819C170BBE0E09FCB5241 Santander +5 0101000020E6100000A6C9EAA399B00DC00A31DF64BD354440 0101000020110F000041C09F2D313719C1EDC9C960A9CB5241 Santander +6 0101000020E6100000D2AD3EB570970DC01F85791E1B354440 0101000020110F00001D0E73E4D22119C14730F65AF4CA5241 BBVA +7 0101000020E6100000B2F9AF3A6FA40DC0BA91F3D44C364440 0101000020110F0000B071A11BDC2C19C16DD5026649CC5241 BBVA +8 0101000020E61000000F1B8134EE8E0DC067D8E65807374440 0101000020110F000033B46EB0981A19C10322017E19CD5241 BBVA +9 0101000020E6100000F12A179193880DC07EBCEDABC2354440 0101000020110F0000C11C4B2F331519C1B2B8FD43AFCB5241 BBVA +10 0101000020E61000004B602775E0A50DC0037374A875364440 0101000020110F00004B5097B1152E19C1481948F276CC5241 BBVA +11 0101000020E6100000F796A5DDEE880DC05337CD960A364440 0101000020110F0000ACE39CB9801519C1F843077FFFCB5241 Santander +12 0101000020E610000041BC5F214A920DC0AFCE2B5507354440 0101000020110F000027682406731D19C15145B148DECA5241 BBVA +13 0101000020E6100000CC2EFD14A0AF0DC00550C5AE47354440 0101000020110F000095956F3A5D3619C11CACED1026CB5241 BBVA +14 0101000020E6100000B113F5FFF28E0DC01075C9419A354440 0101000020110F00005E9EE8C29C1A19C1DB38342E82CB5241 BBVA +15 0101000020E6100000997EFF432F990DC0E6D8677BF1364440 0101000020110F00007E0667274E2319C1ACE7B01801CD5241 Santander +\. + + +-- +-- TOC entry 5787 (class 0 OID 0) +-- Dependencies: 803 +-- Name: analysis_banks_cartodb_id_seq; Type: SEQUENCE SET; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +SELECT pg_catalog.setval('analysis_banks_cartodb_id_seq', 15, true); + + +-- +-- TOC entry 5776 (class 0 OID 13870235) +-- Dependencies: 802 +-- Data for Name: analysis_rent_listings; Type: TABLE DATA; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +COPY analysis_rent_listings (cartodb_id, the_geom, the_geom_webmercator, price) FROM stdin; +1 0101000020E61000006300CEA9C4920DC0FCB8FEFB04354440 0101000020110F0000EA3B5C17DB1D19C135CF17AADBCA5241 81 +2 0101000020E6100000F49AD93201910DC0E37AB970F8354440 0101000020110F00009BD483A95B1C19C11FC4D53FEBCB5241 169 +3 0101000020E610000096F9FBAA0F880DC03974BF03B8354440 0101000020110F00006891BA29C31419C1055F8260A3CB5241 163 +4 0101000020E61000009F08C1D17C930DC0651427FA68364440 0101000020110F00006C35BB7E771E19C1837381CC68CC5241 127 +5 0101000020E61000000A7B7928EAB40DC0425EE239C4364440 0101000020110F00000C6ADB3EDB3A19C14FDB889ACECC5241 58 +6 0101000020E610000095B232CF20A00DC0584C660044354440 0101000020110F0000979587D2332919C177DEB3F521CB5241 184 +7 0101000020E61000001E8C81FF328F0DC031AE47DFB2364440 0101000020110F00006634761DD31A19C13723EB3DBBCC5241 109 +8 0101000020E61000008A014F8FD79D0DC0B1E8EA3376364440 0101000020110F000021A20DC5422719C15A18E08D77CC5241 87 +9 0101000020E61000006ECAA9C393970DC0F71C39950A364440 0101000020110F0000A3016DAAF02119C19C71447DFFCB5241 132 +10 0101000020E6100000FB434A9D57A60DC0F9FD273C3F354440 0101000020110F00008E7BC3E47A2E19C1798082A41CCB5241 134 +11 0101000020E6100000C597512EFE8C0DC032DE0DCDFD344440 0101000020110F0000F51B6C6AF31819C15512CCA6D3CA5241 167 +12 0101000020E610000025E84904589F0DC00BC88CB0BA364440 0101000020110F000024C7054A892819C1C84EBBF6C3CC5241 135 +13 0101000020E61000002917E524E69E0DC0EE4915018C354440 0101000020110F00008011BC93282819C153F8E34772CB5241 160 +14 0101000020E6100000617E8939B1A40DC05EF963BEF0344440 0101000020110F000071896E28142D19C160872616C5CA5241 185 +15 0101000020E6100000B20B809602A30DC0514274FE37354440 0101000020110F00007D63FC6AA62B19C1DC88B19014CB5241 94 +16 0101000020E6100000191978861B9C0DC05C902E6C45354440 0101000020110F0000EBC9ACA6C92519C10A76818B23CB5241 172 +17 0101000020E610000098D6FCD1F2AC0DC01ED4EEFCAE354440 0101000020110F0000FA7E3A3C173419C1DFFAA34E99CB5241 154 +18 0101000020E6100000F9C7CB37DC970DC0B7E914C94E354440 0101000020110F0000795D5C332E2219C1FAAE47FD2DCB5241 183 +19 0101000020E61000002D0263B27B940DC09D26F27AC4354440 0101000020110F0000C4616AF64F1F19C1D2868548B1CB5241 77 +20 0101000020E6100000BD682E39DA8D0DC0DA3ADC937A364440 0101000020110F000038D83D4CAE1919C143A35B6F7CCC5241 196 +21 0101000020E6100000165B41DEB2A90DC0D0D1F568A4354440 0101000020110F00008B9179A8543119C1EAADBA818DCB5241 74 +22 0101000020E6100000DECD65EC4A8D0DC08D309F6B1F354440 0101000020110F000063E4D797341919C1A1034827F9CA5241 164 +23 0101000020E6100000D14DF59EC1AD0DC086C17713C7354440 0101000020110F0000E8ED02DFC63419C15FACD82DB4CB5241 180 +24 0101000020E61000002A41F69FA7900DC063FF9C14AC364440 0101000020110F0000DDF34D960F1C19C1070119AAB3CC5241 62 +25 0101000020E61000001E390026BA8B0DC0B506A19336354440 0101000020110F00005620FE36E01719C109E9F5FB12CB5241 103 +26 0101000020E61000001856B89F9E890DC0AE3A805FA7364440 0101000020110F0000F4E717FF151619C102619069AECC5241 132 +27 0101000020E61000006244348FA7AD0DC093992086E1354440 0101000020110F0000204AB0BCB03419C10E2515AFD1CB5241 64 +28 0101000020E6100000C249D99ADA970DC0D7CF322454364440 0101000020110F00005303A5D42C2219C1463AA98D51CC5241 142 +29 0101000020E61000001E36C089F2930DC0098310FAF2344440 0101000020110F0000E9293E79DB1E19C134C7D593C7CA5241 149 +30 0101000020E610000084DC236A48970DC08AA0D30E37354440 0101000020110F0000A4E5D3ABB02119C12A57638513CB5241 190 +31 0101000020E610000068C42C3D6DAE0DC0E316EF8262354440 0101000020110F000007FF5AA0583519C1CEF97EFE43CB5241 132 +32 0101000020E6100000F0FCDE54BCA70DC03311579A9F354440 0101000020110F0000CE2C83DAA92F19C120A8E62488CB5241 180 +33 0101000020E6100000B5A43D9E809B0DC059B84FF6F1364440 0101000020110F000035B5A016462519C15C07D2A101CD5241 160 +34 0101000020E6100000D4D36C80629E0DC0249F8DFC8B354440 0101000020110F0000F241EAC5B82719C1904ED64272CB5241 95 +35 0101000020E61000008BEA878AF1A50DC05DA77AA5B9354440 0101000020110F0000BBD0E633242E19C15D7E8432A5CB5241 108 +36 0101000020E610000022D3CD1F62B40DC08AC22B852E364440 0101000020110F0000C8E240B6673A19C1DCF9E99427CC5241 168 +37 0101000020E6100000EBAC4CB637950DC0E989CFA0D8364440 0101000020110F0000883BDDA4EF1F19C15478CE5DE5CC5241 80 +38 0101000020E61000007CEFD63A8A960DC006B8BDD1C8364440 0101000020110F00003728B0250F2119C169FD71BAD3CC5241 168 +39 0101000020E6100000756B387CCFA70DC084C36711F7354440 0101000020110F0000CEB4ED1EBA2F19C13F1EE7B7E9CB5241 56 +40 0101000020E6100000A47B2BAC4E960DC0820DC4B760354440 0101000020110F00008BCCAF90DC2019C1AE4248FE41CB5241 162 +41 0101000020E6100000D5645E0C31990DC0F535DB974F364440 0101000020110F00007E8BFFAA4F2319C1B884AA7A4CCC5241 102 +42 0101000020E6100000E3B6E0EA41B40DC02ADE56A899364440 0101000020110F0000FFC4D55B4C3A19C1A602351C9FCC5241 174 +43 0101000020E61000006C513CE4E3B00DC010B3C38659364440 0101000020110F00002817653D703719C14258A88F57CC5241 97 +44 0101000020E61000006B6DF64366A10DC0C980E41EB2364440 0101000020110F0000CFE47B3B482A19C1EFCB4567BACC5241 63 +45 0101000020E61000007031674EA8960DC073B9E40B22354440 0101000020110F0000A608EEB0282119C159033215FCCA5241 159 +46 0101000020E610000002328782CB890DC0A6462A1EAF364440 0101000020110F000008823D1E3C1619C14C749B0DB7CC5241 141 +47 0101000020E61000009694069312A70DC09438822A79354440 0101000020110F0000922DC0AD192F19C113F51A445DCB5241 185 +48 0101000020E610000081F9078F75AE0DC069F4C3ADCD364440 0101000020110F00003F393EB15F3519C1D4F16926D9CC5241 113 +49 0101000020E6100000E348615820AC0DC0D165849A50364440 0101000020110F0000ACEC8F7A643319C14D783D9B4DCC5241 124 +50 0101000020E6100000378F35768FAD0DC085A048F310364440 0101000020110F0000BFF95B459C3419C12A94C89706CC5241 62 +51 0101000020E6100000F25E60BD14B40DC0FA9EFA964F354440 0101000020110F0000EB4840FD253A19C1FF36F6E22ECB5241 109 +52 0101000020E6100000B73BFD3FB6AF0DC0A3D844C97B364440 0101000020110F0000AD1F370E703619C1B9278EC87DCC5241 149 +53 0101000020E6100000FF1181969EB10DC0136E39E5A0364440 0101000020110F0000D10A15CD0E3819C1828B712FA7CC5241 176 +54 0101000020E6100000426F39BD459A0DC0A5505E4CB7354440 0101000020110F000074615DA93A2419C106D4EF93A2CB5241 90 +55 0101000020E6100000E2B0F14972A80DC0AFA4D0BF84354440 0101000020110F00008817D563443019C1202D06306ACB5241 162 +56 0101000020E6100000ADE43CA9F79A0DC0C24A1417D9364440 0101000020110F0000C4944EC5D12419C1D0B5C2E1E5CC5241 80 +57 0101000020E6100000AD6B99A9818B0DC07E06699A33364440 0101000020110F00007EE2C43DB01719C12BBF9D402DCC5241 172 +58 0101000020E610000098272DE67B960DC08EC657CCF4344440 0101000020110F0000352CE4F9022119C16660F49BC9CA5241 177 +59 0101000020E6100000DEAA210E56A90DC0B45845C72B354440 0101000020110F00003689FED4053119C1F0FE50F006CB5241 172 +60 0101000020E61000004CAD1CF03B8F0DC07A4C4F1EED364440 0101000020110F000048EE2CB5DA1A19C1E236513AFCCC5241 122 +\. + + +-- +-- TOC entry 5788 (class 0 OID 0) +-- Dependencies: 801 +-- Name: analysis_rent_listings_cartodb_id_seq; Type: SEQUENCE SET; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +SELECT pg_catalog.setval('analysis_rent_listings_cartodb_id_seq', 60, true); + + +-- +-- TOC entry 5618 (class 2606 OID 13870260) +-- Name: analysis_banks_pkey; Type: CONSTRAINT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +ALTER TABLE ONLY analysis_banks + ADD CONSTRAINT analysis_banks_pkey PRIMARY KEY (cartodb_id); + + +-- +-- TOC entry 5614 (class 2606 OID 13870243) +-- Name: analysis_rent_listings_pkey; Type: CONSTRAINT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +ALTER TABLE ONLY analysis_rent_listings + ADD CONSTRAINT analysis_rent_listings_pkey PRIMARY KEY (cartodb_id); + + +-- +-- TOC entry 5619 (class 1259 OID 13870261) +-- Name: analysis_banks_the_geom_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_banks_the_geom_idx ON analysis_banks USING gist (the_geom); + + +-- +-- TOC entry 5620 (class 1259 OID 13870262) +-- Name: analysis_banks_the_geom_webmercator_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_banks_the_geom_webmercator_idx ON analysis_banks USING gist (the_geom_webmercator); + + +-- +-- TOC entry 5615 (class 1259 OID 13870244) +-- Name: analysis_rent_listings_the_geom_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_rent_listings_the_geom_idx ON analysis_rent_listings USING gist (the_geom); + + +-- +-- TOC entry 5616 (class 1259 OID 13870245) +-- Name: analysis_rent_listings_the_geom_webmercator_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_rent_listings_the_geom_webmercator_idx ON analysis_rent_listings USING gist (the_geom_webmercator); + + +-- +-- TOC entry 5623 (class 2620 OID 13870248) +-- Name: test_quota; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON analysis_rent_listings FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_checkquota('0.1', '-1', 'public'); + + +-- +-- TOC entry 5627 (class 2620 OID 13870265) +-- Name: test_quota; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON analysis_banks FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_checkquota('0.1', '-1', 'public'); + + +-- +-- TOC entry 5624 (class 2620 OID 13870249) +-- Name: test_quota_per_row; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON analysis_rent_listings FOR EACH ROW EXECUTE PROCEDURE cartodb.cdb_checkquota('0.001', '-1', 'public'); + + +-- +-- TOC entry 5628 (class 2620 OID 13870266) +-- Name: test_quota_per_row; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON analysis_banks FOR EACH ROW EXECUTE PROCEDURE cartodb.cdb_checkquota('0.001', '-1', 'public'); + + +-- +-- TOC entry 5621 (class 2620 OID 13870246) +-- Name: track_updates; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON analysis_rent_listings FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); + + +-- +-- TOC entry 5625 (class 2620 OID 13870263) +-- Name: track_updates; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON analysis_banks FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); + + +-- +-- TOC entry 5622 (class 2620 OID 13870247) +-- Name: update_the_geom_webmercator_trigger; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON analysis_rent_listings FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); + + +-- +-- TOC entry 5626 (class 2620 OID 13870264) +-- Name: update_the_geom_webmercator_trigger; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON analysis_banks FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); + + +-- +-- TOC entry 5783 (class 0 OID 0) +-- Dependencies: 804 +-- Name: analysis_banks; Type: ACL; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +REVOKE ALL ON TABLE analysis_banks FROM PUBLIC; +REVOKE ALL ON TABLE analysis_banks FROM :TESTUSER; +GRANT ALL ON TABLE analysis_banks TO :TESTUSER; +GRANT SELECT ON TABLE analysis_banks TO :PUBLICUSER; + + +-- +-- TOC entry 5785 (class 0 OID 0) +-- Dependencies: 802 +-- Name: analysis_rent_listings; Type: ACL; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +REVOKE ALL ON TABLE analysis_rent_listings FROM PUBLIC; +REVOKE ALL ON TABLE analysis_rent_listings FROM :TESTUSER; +GRANT ALL ON TABLE analysis_rent_listings TO :TESTUSER; +GRANT SELECT ON TABLE analysis_rent_listings TO :PUBLICUSER; + + +-- Completed on 2016-02-29 12:50:53 CET + +-- +-- PostgreSQL database dump complete +-- + From 3b11525cfb302e9cc48b567ed3c6b54754af12a5 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 2 Mar 2016 12:43:14 +0100 Subject: [PATCH 12/80] Add analysis use cases that we need to support --- .../analysis/analysis-layers-use-cases.js | 653 ++++++++++++++++++ 1 file changed, 653 insertions(+) create mode 100644 test/acceptance/analysis/analysis-layers-use-cases.js diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js new file mode 100644 index 00000000..b74446aa --- /dev/null +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -0,0 +1,653 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); +var dot = require('dot'); + +describe('analysis-layers use cases', function() { + + + var multitypeStyleTemplate = dot.template([ + "#points['mapnik::geometry_type'=1] {", + " marker-fill-opacity: {{=it._opacity}};", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: {{=it._opacity}};", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: {{=it._color}};", + " marker-allow-overlap: true;", + "}", + "#lines['mapnik::geometry_type'=2] {", + " line-color: {{=it._color}};", + " line-width: 2;", + " line-opacity: {{=it._opacity}};", + "}", + "#polygons['mapnik::geometry_type'=3] {", + " polygon-fill: {{=it._color}};", + " polygon-opacity: {{=it._opacity}};", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: {{=it._opacity}};", + "}" + ].join('\n')); + + + function cartocss(color, opacity) { + return multitypeStyleTemplate({ + _color: color || '#F11810', + _opacity: Number.isFinite(opacity) ? opacity : 1 + }); + } + + function mapConfig(layers, analysis) { + return { + version: '1.5.0', + layers: layers, + analysis: analysis || [] + }; + } + + function analysisDef(analysis) { + return JSON.stringify(analysis); + } + + var DEFAULT_MULTITYPE_STYLE = cartocss(); + + var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 }; + + var useCases = [ + { + desc: '1 mapnik layer', + mapConfig: { + version: '1.5.0', + layers: [ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + } + ] + } + }, + + { + desc: '2 mapnik layers', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_banks", + cartocss: cartocss('#2167AB'), + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + } + ]) + }, + + { + desc: 'rent listings + buffer over atm-machines', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radio": 250 + } + }), + cartocss: cartocss('black', 0.5) + } + } + ]) + }, + + { + desc: 'rent listings + point-in-polygon from buffer atm-machines and rent listings', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "point-in-polygon", + "params": { + "pointsSource": { + "type": "source", + "params": { + "query": "select * from analysis_rent_listings" + } + }, + "polygonsSource": { + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radio": 250 + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + } + ]) + }, + + { + desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings', + mapConfig: mapConfig([ + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "point-in-polygon", + "params": { + "pointsSource": { + "type": "source", + "params": { + "query": "select * from analysis_rent_listings" + } + }, + "polygonsSource": { + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radio": 250 + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + }, + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + } + ]) + }, + + { + desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radio": 300 + } + }), + cartocss: cartocss('magenta', 0.5) + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "point-in-polygon", + "params": { + "pointsSource": { + "type": "source", + "params": { + "query": "select * from analysis_rent_listings" + } + }, + "polygonsSource": { + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radio": 300 + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + } + ]) + }, + + { + skip: true, + desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + "source": { id: "a" }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "b1" }, + "cartocss": cartocss('green', 1.0), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "b2" }, + "cartocss": cartocss('magenta', 0.5), + "cartocss_version": "2.3.0" + } + } + ], + [ + { + id: "b2", + options: { + def: analysisDef({ + "type": "count-in-polygon", + "id": "a0", + "params": { + "columnName": 'count_airbnb', + "pointsSource": { + "type": "source", + "params": { + query: "select * from analysis_rent_listings" + }, + dataviews: { + price_histogram: { + type: 'histogram', + options: { + column: 'price' + } + } + } + }, + "polygonsSource": { + "id": "b1", + "type": "buffer", + "params": { + "source": { + "id": "b0", + "type": "source", + "params": { + query: "select * from analysis_banks" + } + }, + "radio": 250 + }, + dataviews: { + bank_category: { + type: 'aggregation', + options: { + column: 'bank' + } + } + } + } + }, + dataviews: { + count_histogram: { + type: 'histogram', + options: { + column: 'count_airbnb' + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + } + ]) + }, + + { + skip: true, + desc: 'I. Distribution centers', + mapConfig: mapConfig( + [ + { + type: 'cartodb', + options: { + "source": { id: "b0" }, + "cartocss": [ + "#distribution_centers {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 0.7;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: blue;", + " marker-allow-overlap: true;", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "a0" }, + "cartocss": [ + "#shops {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 0.7;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: red;", + " marker-allow-overlap: true;", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "a1" }, + "cartocss": [ + "#routing {", + " line-color: ramp([routing_time], colorbrewer(Reds));", + " line-width: ramp([routing_time], 2, 8);", + " line-opacity: 1.0;", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + } + ], + [ + { + id: 'a1', + type: 'routing-n-to-n', + params: { + // distanceColumn: 'routing_distance', + // timeColumn: 'routing_time', + originSource: { + id: 'b0', + type: 'source', + params: { + query: 'select * from distribution_centers' + }, + dataviews: { + distribution_center_name_category: { + type: 'aggregation', + options: { + column: 'name' + } + } + } + }, + destinationSource: { + id: 'a0', + type: 'source', + params: { + query: 'select * from shops' + } + } + }, + dataviews: { + time_histogram: { + type: 'histogram', + options: { + column: 'routing_time' + } + }, + distance_histogram: { + type: 'histogram', + options: { + column: 'routing_distance' + } + } + } + } + ] + ) + }, + + { + skip: true, + desc: 'II. Population analysis', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + "source": { id: "b2" }, + "cartocss": [ + "#count_in_polygon {", + " polygon-opacity: 1.0", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: 0.7", + " polygon-fill: ramp([estimated_people], colorbrewer(Reds));", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "b0" }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + } + ], + [ + { + id: 'b3', + type: 'union', + params: { + join_on: 'cartodb_id', + source: { + id: 'b2', + type: 'estimated-population', + params: { + columnName: 'estimated_people', + source: { + id: 'b1', + type: 'trade-area', + params: { + source: { + "id": "b0", + "type": "source", + "params": { + query: "select * from subway_stops" + } + } + }, + dataviews: { + subway_line_category: { + type: 'aggregation', + options: { + column: 'subway_line' + } + } + } + } + }, + dataviews: { + people_histogram: { + type: 'histogram', + options: { + column: 'estimated_people' + } + } + } + } + }, + dataviews: { + total_population_formula: { + type: 'formula', + options: { + column: 'estimated_people', + operation: 'sum' + } + } + } + } + ]) + }, + + { + skip: true, + desc: 'III. Point in polygon', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + "source": { id: "a1" }, + "cartocss": [ + "#count_in_polygon {", + " polygon-opacity: 1.0", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: 0.7", + " polygon-fill: ramp([count_people], colorbrewer(Reds));", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + } + ], + [ + { + "id": "a1", + "type": "count-in-polygon", + "params": { + "columnName": 'count_people', + "pointsSource": { + "id": 'a0', + "type": "source", + "params": { + query: "select the_geom, age, gender, income from analysis_rent_listings" + }, + dataviews: { + age_histogram: { + type: 'histogram', + options: { + column: 'age' + } + }, + income_histogram: { + type: 'histogram', + options: { + column: 'income' + } + }, + gender_category: { + type: 'aggregation', + options: { + column: 'gender' + } + } + } + }, + "polygonsSource": { + "id": "b0", + "type": "source", + "params": { + query: "select * from postal_codes" + } + } + } + } + ] + ) + } + + ]; + + useCases.forEach(function(useCase, imageIdx) { + if (!!useCase.skip) { + console.log(JSON.stringify(useCase.mapConfig, null, 4)); + } + it.skip('should implement use case: "' + useCase.desc + '"', function(done) { + + var testClient = new TestClient(useCase.mapConfig, 1234); + + var tile = useCase.tile || TILE_ANALYSIS_TABLES; + + testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { + assert.ok(!err, err); + + image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); + + assert.equal(image.width(), 256); + + testClient.drain(done); + }); + }); + }); +}); From 9b9e6b13b72503d0daab1d24cbfbfc76f6e039ca Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 2 Mar 2016 13:27:28 +0100 Subject: [PATCH 13/80] Fix query table --- test/acceptance/analysis/analysis-layers-use-cases.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index b74446aa..23ba98f8 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -591,7 +591,7 @@ describe('analysis-layers use cases', function() { "id": 'a0', "type": "source", "params": { - query: "select the_geom, age, gender, income from analysis_rent_listings" + query: "select the_geom, age, gender, income from people" }, dataviews: { age_histogram: { From 16654c016a9ff7c88a25440ebd41f14fe3f14f10 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 2 Mar 2016 13:27:45 +0100 Subject: [PATCH 14/80] Style --- test/acceptance/analysis/analysis-layers-use-cases.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 23ba98f8..f19114b2 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -563,7 +563,8 @@ describe('analysis-layers use cases', function() { { skip: true, desc: 'III. Point in polygon', - mapConfig: mapConfig([ + mapConfig: mapConfig( + [ { type: 'cartodb', options: { From 011b60eeab0367870aba5bf3658077cf7efa0291 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 2 Mar 2016 13:27:53 +0100 Subject: [PATCH 15/80] Change ids --- .../acceptance/analysis/analysis-layers-use-cases.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index f19114b2..8e6fca23 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -482,7 +482,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - "source": { id: "b2" }, + "source": { id: "a2" }, "cartocss": [ "#count_in_polygon {", " polygon-opacity: 1.0", @@ -498,7 +498,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - "source": { id: "b0" }, + "source": { id: "a0" }, "cartocss": DEFAULT_MULTITYPE_STYLE, "cartocss_version": "2.3.0" } @@ -506,21 +506,21 @@ describe('analysis-layers use cases', function() { ], [ { - id: 'b3', + id: 'a3', type: 'union', params: { join_on: 'cartodb_id', source: { - id: 'b2', + id: 'a2', type: 'estimated-population', params: { columnName: 'estimated_people', source: { - id: 'b1', + id: 'a1', type: 'trade-area', params: { source: { - "id": "b0", + "id": "a0", "type": "source", "params": { query: "select * from subway_stops" From 2eac808e186665927d1b191ed958a4190142be66 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 3 Mar 2016 11:45:37 +0100 Subject: [PATCH 16/80] Change analysis name so it's easier to understand --- test/acceptance/analysis/analysis-layers-use-cases.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 8e6fca23..d7d9e5e1 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -507,9 +507,8 @@ describe('analysis-layers use cases', function() { [ { id: 'a3', - type: 'union', + type: 'total-population', params: { - join_on: 'cartodb_id', source: { id: 'a2', type: 'estimated-population', From 69142964c691f7c4e3ea464e2707d4b7b09b8503 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 3 Mar 2016 11:54:50 +0100 Subject: [PATCH 17/80] fix trade-area params --- test/acceptance/analysis/analysis-layers-use-cases.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index d7d9e5e1..2f95ceea 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -524,7 +524,9 @@ describe('analysis-layers use cases', function() { "params": { query: "select * from subway_stops" } - } + }, + kind: 'walk', + time: 300 }, dataviews: { subway_line_category: { From 31dede5d06aeaa8eb6c8c54ff491ea242b6cbded Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 3 Mar 2016 12:01:49 +0100 Subject: [PATCH 18/80] Notes to make clear the total-population analysis --- test/acceptance/analysis/analysis-layers-use-cases.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 2f95ceea..5657741c 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -507,8 +507,10 @@ describe('analysis-layers use cases', function() { [ { id: 'a3', + // this will union the polygons, produce just one polygon, and calculate the total population for it type: 'total-population', params: { + columnName: 'total_population', source: { id: 'a2', type: 'estimated-population', From e53d823b5a12917dcfaa7ca4a7d6d151b726bc46 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 3 Mar 2016 12:04:03 +0100 Subject: [PATCH 19/80] Fix total population column name for widget --- test/acceptance/analysis/analysis-layers-use-cases.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 5657741c..4879a380 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -554,7 +554,7 @@ describe('analysis-layers use cases', function() { total_population_formula: { type: 'formula', options: { - column: 'estimated_people', + column: 'total_population', operation: 'sum' } } From f9c0e29db0f94bf5f2823e04d450ef75fe39b779 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 3 Mar 2016 11:47:52 +0100 Subject: [PATCH 20/80] Dataviews separated from analysis They are just another consumer of the analysis as layers are. --- .../analysis/analysis-layers-use-cases.js | 152 ++++++++++-------- 1 file changed, 83 insertions(+), 69 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 4879a380..6084b88c 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -41,10 +41,11 @@ describe('analysis-layers use cases', function() { }); } - function mapConfig(layers, analysis) { + function mapConfig(layers, dataviews, analysis) { return { version: '1.5.0', layers: layers, + dataviews: dataviews || {}, analysis: analysis || [] }; } @@ -370,6 +371,7 @@ describe('analysis-layers use cases', function() { skip: true, desc: 'I. Distribution centers', mapConfig: mapConfig( + // layers [ { type: 'cartodb', @@ -426,6 +428,31 @@ describe('analysis-layers use cases', function() { } } ], + // dataviews + { + distribution_center_name_category: { + source: { id: 'b0' }, + type: 'aggregation', + options: { + column: 'name' + } + }, + time_histogram: { + source: { id: 'a1' }, + type: 'histogram', + options: { + column: 'routing_time' + } + }, + distance_histogram: { + source: { id: 'a1' }, + type: 'histogram', + options: { + column: 'routing_distance' + } + } + }, + // analysis [ { id: 'a1', @@ -438,14 +465,6 @@ describe('analysis-layers use cases', function() { type: 'source', params: { query: 'select * from distribution_centers' - }, - dataviews: { - distribution_center_name_category: { - type: 'aggregation', - options: { - column: 'name' - } - } } }, destinationSource: { @@ -455,20 +474,6 @@ describe('analysis-layers use cases', function() { query: 'select * from shops' } } - }, - dataviews: { - time_histogram: { - type: 'histogram', - options: { - column: 'routing_time' - } - }, - distance_histogram: { - type: 'histogram', - options: { - column: 'routing_distance' - } - } } } ] @@ -478,7 +483,9 @@ describe('analysis-layers use cases', function() { { skip: true, desc: 'II. Population analysis', - mapConfig: mapConfig([ + mapConfig: mapConfig( + // layers + [ { type: 'cartodb', options: { @@ -504,6 +511,32 @@ describe('analysis-layers use cases', function() { } } ], + // dataviews + { + total_population_formula: { + "source": { id: "a3" }, + type: 'formula', + options: { + column: 'total_population', + operation: 'sum' + } + }, + people_histogram: { // this injects a range filter at `a2` node output + "source": { id: "a2" }, + type: 'histogram', + options: { + column: 'estimated_people' + } + }, + subway_line_category: { // this injects a category filter at `a0` node output + "source": { id: "a0" }, + type: 'aggregation', + options: { + column: 'subway_line' + } + } + }, + // analysis [ { id: 'a3', @@ -529,33 +562,8 @@ describe('analysis-layers use cases', function() { }, kind: 'walk', time: 300 - }, - dataviews: { - subway_line_category: { - type: 'aggregation', - options: { - column: 'subway_line' - } - } } } - }, - dataviews: { - people_histogram: { - type: 'histogram', - options: { - column: 'estimated_people' - } - } - } - } - }, - dataviews: { - total_population_formula: { - type: 'formula', - options: { - column: 'total_population', - operation: 'sum' } } } @@ -567,6 +575,7 @@ describe('analysis-layers use cases', function() { skip: true, desc: 'III. Point in polygon', mapConfig: mapConfig( + // layers [ { type: 'cartodb', @@ -585,6 +594,31 @@ describe('analysis-layers use cases', function() { } } ], + // dataviews + { + age_histogram: { + "source": { id: "a0" }, + type: 'histogram', + options: { + column: 'age' + } + }, + income_histogram: { + "source": { id: "a0" }, + type: 'histogram', + options: { + column: 'income' + } + }, + gender_category: { + "source": { id: "a0" }, + type: 'aggregation', + options: { + column: 'gender' + } + } + }, + // analysis [ { "id": "a1", @@ -596,26 +630,6 @@ describe('analysis-layers use cases', function() { "type": "source", "params": { query: "select the_geom, age, gender, income from people" - }, - dataviews: { - age_histogram: { - type: 'histogram', - options: { - column: 'age' - } - }, - income_histogram: { - type: 'histogram', - options: { - column: 'income' - } - }, - gender_category: { - type: 'aggregation', - options: { - column: 'gender' - } - } } }, "polygonsSource": { From f504807812b0dc364dce43c9518fa606b21c6823 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 4 Mar 2016 12:12:12 +0100 Subject: [PATCH 21/80] Regenerate npm-shrinkwrap.json --- npm-shrinkwrap.json | 146 +++++++++++++++++++++++++++----------------- 1 file changed, 89 insertions(+), 57 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 1ea73fe8..28305dbd 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -74,9 +74,9 @@ } }, "type-is": { - "version": "1.6.11", + "version": "1.6.12", "from": "type-is@>=1.6.10 <1.7.0", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.11.tgz", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { "version": "0.3.0", @@ -84,14 +84,14 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" }, "mime-types": { - "version": "2.1.9", - "from": "mime-types@>=2.1.9 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "version": "2.1.10", + "from": "mime-types@>=2.1.2 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } } @@ -144,9 +144,28 @@ "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz" }, "aws4": { - "version": "1.2.1", + "version": "1.3.2", "from": "aws4@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.2.1.tgz" + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz", + "dependencies": { + "lru-cache": { + "version": "4.0.0", + "from": "lru-cache@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz", + "dependencies": { + "pseudomap": { + "version": "1.0.2", + "from": "pseudomap@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz" + }, + "yallist": { + "version": "2.0.0", + "from": "yallist@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.0.0.tgz" + } + } + } + } }, "bl": { "version": "1.0.3", @@ -235,14 +254,21 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz", "dependencies": { "ansi-styles": { - "version": "2.1.0", + "version": "2.2.0", "from": "ansi-styles@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.0.tgz", + "dependencies": { + "color-convert": { + "version": "1.0.0", + "from": "color-convert@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.0.0.tgz" + } + } }, "escape-string-regexp": { - "version": "1.0.4", + "version": "1.0.5", "from": "escape-string-regexp@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" }, "has-ansi": { "version": "2.0.0", @@ -257,9 +283,9 @@ } }, "strip-ansi": { - "version": "3.0.0", + "version": "3.0.1", "from": "strip-ansi@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "dependencies": { "ansi-regex": { "version": "2.0.0", @@ -288,9 +314,9 @@ } }, "is-my-json-valid": { - "version": "2.12.4", + "version": "2.13.1", "from": "is-my-json-valid@>=2.12.4 <3.0.0", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.4.tgz", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz", "dependencies": { "generate-function": { "version": "2.0.0", @@ -422,9 +448,9 @@ "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz" }, "tweetnacl": { - "version": "0.13.3", + "version": "0.14.1", "from": "tweetnacl@>=0.13.0 <1.0.0", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.13.3.tgz" + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.1.tgz" }, "jodid25519": { "version": "1.0.2", @@ -525,6 +551,11 @@ } } }, + "cartodb-query-tables": { + "version": "0.1.0", + "from": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master", + "resolved": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" + }, "cartodb-redis": { "version": "0.13.0", "from": "cartodb-redis@>=0.13.0 <0.14.0", @@ -594,14 +625,14 @@ "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz", "dependencies": { "mime-types": { - "version": "2.1.9", - "from": "mime-types@>=2.1.6 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "version": "2.1.10", + "from": "mime-types@>=2.1.10 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } }, @@ -659,7 +690,7 @@ "dependencies": { "unpipe": { "version": "1.0.0", - "from": "unpipe@>=1.0.0 <1.1.0", + "from": "unpipe@1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" } } @@ -773,9 +804,9 @@ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.2.tgz" }, "type-is": { - "version": "1.6.11", - "from": "type-is@>=1.6.6 <1.7.0", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.11.tgz", + "version": "1.6.12", + "from": "type-is@>=1.6.10 <1.7.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { "version": "0.3.0", @@ -783,14 +814,14 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" }, "mime-types": { - "version": "2.1.9", - "from": "mime-types@>=2.1.6 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "version": "2.1.10", + "from": "mime-types@>=2.1.10 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } } @@ -872,11 +903,6 @@ "from": "lzma@>=1.3.7 <1.4.0", "resolved": "https://registry.npmjs.org/lzma/-/lzma-1.3.7.tgz" }, - "cartodb-query-tables": { - "version": "0.1.0", - "from": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master", - "resolved": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" - }, "node-statsd": { "version": "0.0.7", "from": "node-statsd@>=0.0.7 <0.1.0", @@ -927,9 +953,9 @@ "resolved": "https://registry.npmjs.org/request/-/request-2.62.0.tgz", "dependencies": { "bl": { - "version": "1.0.2", + "version": "1.0.3", "from": "bl@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz", "dependencies": { "readable-stream": { "version": "2.0.5", @@ -1003,14 +1029,14 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" }, "mime-types": { - "version": "2.1.9", + "version": "2.1.10", "from": "mime-types@>=2.1.2 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } }, @@ -1131,14 +1157,21 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz", "dependencies": { "ansi-styles": { - "version": "2.1.0", + "version": "2.2.0", "from": "ansi-styles@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.0.tgz", + "dependencies": { + "color-convert": { + "version": "1.0.0", + "from": "color-convert@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.0.0.tgz" + } + } }, "escape-string-regexp": { - "version": "1.0.4", + "version": "1.0.5", "from": "escape-string-regexp@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" }, "has-ansi": { "version": "2.0.0", @@ -1153,9 +1186,9 @@ } }, "strip-ansi": { - "version": "3.0.0", + "version": "3.0.1", "from": "strip-ansi@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "dependencies": { "ansi-regex": { "version": "2.0.0", @@ -1184,9 +1217,9 @@ } }, "is-my-json-valid": { - "version": "2.12.4", + "version": "2.13.1", "from": "is-my-json-valid@>=2.12.0 <3.0.0", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.4.tgz", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz", "dependencies": { "generate-function": { "version": "2.0.0", @@ -4132,7 +4165,6 @@ "turbo-cartocss": { "version": "0.4.0", "from": "turbo-cartocss@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/turbo-cartocss/-/turbo-cartocss-0.4.0.tgz", "dependencies": { "browser-request": { "version": "0.3.3", From 634a4c2a0141d5b47e46aa01c5180f48d005ace6 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 4 Mar 2016 16:20:23 +0100 Subject: [PATCH 22/80] Debug option for internal nodes: it allows to display and customize cartocss --- .../models/mapconfig_analysis_layers_adapter.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js index 2b000ecc..cce72595 100644 --- a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js +++ b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js @@ -104,11 +104,13 @@ MapConfigAnalysisLayersAdapter.prototype.getLayers = function(analysisConfigurat var layers = []; analysis.getSortedNodes().reverse().forEach(function(node, i) { - var layer = getLayer(node.getQuery(), node.getColumns(), '#333', 0.1 * i); - if (!!node.params && !!node.params.cartocss) { - layer = getLayerWithStyle(node.getQuery(), node.getColumns(), node.params.cartocss); + if (node.params && node.params.debug) { + var layer = getLayer(node.getQuery(), node.getColumns(), '#333', 0.1 * i); + if (!!node.params && !!node.params.cartocss) { + layer = getLayerWithStyle(node.getQuery(), node.getColumns(), node.params.cartocss); + } + layers.push(layer); } - layers.push(layer); }); layers.push(getLayerWithStyle( From 9d77aea6be8507c8878b1e80485993baa7c66664 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 8 Mar 2016 15:23:56 +0100 Subject: [PATCH 23/80] Regenerate npm-shrinkwrap.json --- npm-shrinkwrap.json | 64 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 28305dbd..5defa603 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -62,10 +62,15 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-5.2.0.tgz" }, "raw-body": { - "version": "2.1.5", + "version": "2.1.6", "from": "raw-body@>=2.1.5 <2.2.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.5.tgz", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.6.tgz", "dependencies": { + "bytes": { + "version": "2.3.0", + "from": "bytes@2.3.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.3.0.tgz" + }, "unpipe": { "version": "1.0.0", "from": "unpipe@1.0.0", @@ -75,7 +80,7 @@ }, "type-is": { "version": "1.6.12", - "from": "type-is@>=1.6.10 <1.7.0", + "from": "type-is@>=1.6.6 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { @@ -85,7 +90,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.2 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -106,7 +111,7 @@ "dependencies": { "async": { "version": "1.5.2", - "from": "async@>=1.5.2 <2.0.0", + "from": "async@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" }, "cartodb-psql": { @@ -626,7 +631,7 @@ "dependencies": { "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -690,7 +695,7 @@ "dependencies": { "unpipe": { "version": "1.0.0", - "from": "unpipe@1.0.0", + "from": "unpipe@>=1.0.0 <1.1.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" } } @@ -805,7 +810,7 @@ }, "type-is": { "version": "1.6.12", - "from": "type-is@>=1.6.10 <1.7.0", + "from": "type-is@>=1.6.6 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { @@ -815,7 +820,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -4182,9 +4187,9 @@ "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.1.2.tgz" }, "less": { - "version": "2.6.0", + "version": "2.6.1", "from": "less@>=2.5.3 <3.0.0", - "resolved": "https://registry.npmjs.org/less/-/less-2.6.0.tgz", + "resolved": "https://registry.npmjs.org/less/-/less-2.6.1.tgz", "dependencies": { "errno": { "version": "0.1.4", @@ -4199,14 +4204,14 @@ } }, "graceful-fs": { - "version": "3.0.8", - "from": "graceful-fs@>=3.0.5 <4.0.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.8.tgz" + "version": "4.1.3", + "from": "graceful-fs@>=4.1.2 <5.0.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.3.tgz" }, "image-size": { - "version": "0.3.5", - "from": "image-size@>=0.3.5 <0.4.0", - "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.3.5.tgz" + "version": "0.4.0", + "from": "image-size@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.4.0.tgz" }, "mime": { "version": "1.3.4", @@ -4226,28 +4231,21 @@ } }, "promise": { - "version": "6.1.0", - "from": "promise@>=6.0.1 <7.0.0", - "resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz", + "version": "7.1.1", + "from": "promise@>=7.1.1 <8.0.0", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.1.1.tgz", "dependencies": { "asap": { - "version": "1.0.0", - "from": "asap@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz" + "version": "2.0.3", + "from": "asap@>=2.0.3 <2.1.0", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.3.tgz" } } }, "source-map": { - "version": "0.4.4", - "from": "source-map@>=0.4.2 <0.5.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", - "dependencies": { - "amdefine": { - "version": "1.0.0", - "from": "amdefine@>=0.0.4", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.0.tgz" - } - } + "version": "0.5.3", + "from": "source-map@>=0.5.3 <0.6.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz" } } }, From 7fa8d1e0c93d258d48c953b079904860586ec4f1 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 9 Mar 2016 17:39:20 +0100 Subject: [PATCH 24/80] Analyses are now an array and layers consume from their nodes Layers now can define a `source: {id: 'a0'}` option to point to an analysis node that will be used as the query for that layer. --- .../mapconfig_analysis_layers_adapter.js | 146 ++++-------------- 1 file changed, 33 insertions(+), 113 deletions(-) diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js index cce72595..de3ea242 100644 --- a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js +++ b/lib/cartodb/models/mapconfig_analysis_layers_adapter.js @@ -1,5 +1,4 @@ var queue = require('queue-async'); -var _ = require('underscore'); var camshaft = require('camshaft'); var dot = require('dot'); @@ -21,39 +20,6 @@ function skipColumns(columnNames) { .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; }); } -var multitypeStyleTemplate = dot.template([ - "#points['mapnik::geometry_type'=1] {", - " marker-fill-opacity: {{=it._opacity}};", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: {{=it._opacity}};", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 4;", - " marker-fill: {{=it._color}};", - " marker-allow-overlap: true;", - "}", - "#lines['mapnik::geometry_type'=2] {", - " line-color: {{=it._color}};", - " line-width: 2;", - " line-opacity: {{=it._opacity}};", - "}", - "#polygons['mapnik::geometry_type'=3] {", - " polygon-fill: {{=it._color}};", - " polygon-opacity: {{=it._opacity}};", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: {{=it._opacity}};", - "}" -].join('\n')); - -function multiTypeStyle(color, opacity) { - return multitypeStyleTemplate({ - _opacity: opacity || 1.0, - _color: color || 'red' - }); -} - var layerQueryTemplate = dot.template([ 'SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, {{=it._columns}}', 'FROM ({{=it._query}}) _cdb_analysis_query' @@ -63,100 +29,54 @@ function layerQuery(query, columnNames) { return layerQueryTemplate({ _query: query, _columns: skipColumns(columnNames).join(', ') }); } -function getLayer(query, columnNames, color, opacity) { - return { - type: 'mapnik', - options: { - sql: layerQuery(query, columnNames), - cartocss: multiTypeStyle(color, opacity), - cartocss_version: '2.3.0' - } - }; -} - -function getLayerWithStyle(query, columnNames, cartocss) { - return { - type: 'mapnik', - options: { - sql: layerQuery(query, columnNames), - cartocss: cartocss, - cartocss_version: '2.3.0' - } - }; +function shouldAdaptLayers(requestMapConfig) { + return Array.isArray(requestMapConfig.layers) && + Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; } MapConfigAnalysisLayersAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, callback) { - if (!Array.isArray(requestMapConfig.layers)) { + if (!shouldAdaptLayers(requestMapConfig)) { return callback(null, requestMapConfig); } - function adaptLayer(layer, done) { - if (isAnalysisTypeLayer(layer)) { - - var analysisDefinition = JSON.parse(layer.options.def); - - camshaft.create(analysisConfiguration, analysisDefinition, function(err, analysis) { - if (err) { - return done(err); - } - - var layers = []; - - analysis.getSortedNodes().reverse().forEach(function(node, i) { - if (node.params && node.params.debug) { - var layer = getLayer(node.getQuery(), node.getColumns(), '#333', 0.1 * i); - if (!!node.params && !!node.params.cartocss) { - layer = getLayerWithStyle(node.getQuery(), node.getColumns(), node.params.cartocss); - } - layers.push(layer); - } - }); - - layers.push(getLayerWithStyle( - analysis.getQuery(), - analysis.getRoot().getColumns(), - layer.options.cartocss - )); - - return done(null, { layers: layers }); - }); - } else { - return done(null, { layers: [layer] }); - } + function createAnalysis(analysisDefinition, done) { + camshaft.create(analysisConfiguration, analysisDefinition, done); } - function layersAdaptQueueFinish(err, layersResults) { + var analysesQueue = queue(requestMapConfig.analyses.length); + requestMapConfig.analyses.forEach(function(analysis) { + analysesQueue.defer(createAnalysis, analysis); + }); + + analysesQueue.awaitAll(function(err, analysesResults) { if (err) { return callback(err); } - if (!layersResults || layersResults.length === 0) { - return callback(new Error('Missing layers array from layergroup config')); - } + var sourceId2Node = analysesResults.reduce(function(sourceId2Query, analysis) { + var rootNode = analysis.getRoot(); + if (rootNode.params && rootNode.params.id) { + sourceId2Query[rootNode.params.id] = rootNode; + } - var layers = []; - layersResults.forEach(function(layersResult) { - layers = layers.concat(layersResult.layers); + analysis.getSortedNodes().forEach(function(node) { + if (node.params && node.params.id) { + sourceId2Query[node.params.id] = node; + } + }); + + return sourceId2Query; + }, {}); + + requestMapConfig.layers = requestMapConfig.layers.map(function(layer) { + if (layer.options.source && layer.options.source.id) { + var layerNode = sourceId2Node[layer.options.source.id]; + layer.options.sql = layerQuery(layerNode.getQuery(), layerNode.getColumns()); + } + return layer; }); - requestMapConfig.layers = layers; - return callback(null, requestMapConfig); - } - - var adaptLayersQueue = queue(requestMapConfig.layers.length); - - if (_.some(requestMapConfig.layers, isAnalysisTypeLayer)) { - requestMapConfig.layers.forEach(function(layer) { - adaptLayersQueue.defer(adaptLayer, layer); - }); - adaptLayersQueue.awaitAll(layersAdaptQueueFinish); - } else { - return callback(null, requestMapConfig); - } + }); }; - -function isAnalysisTypeLayer(layer) { - return layer.type === 'analysis'; -} From 1535820d4f2d21af751c80efd42e16a8e9181dbd Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Mar 2016 11:32:43 +0100 Subject: [PATCH 25/80] Regenerate npm-shrinkwrap.json --- npm-shrinkwrap.json | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 851d8332..4e11816a 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -111,7 +111,7 @@ "dependencies": { "async": { "version": "1.5.2", - "from": "async@>=1.5.2 <2.0.0", + "from": "async@>=1.4.0 <2.0.0", "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" }, "cartodb-psql": { @@ -519,9 +519,9 @@ "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz" }, "tough-cookie": { - "version": "2.2.1", + "version": "2.2.2", "from": "tough-cookie@>=2.2.0 <2.3.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.1.tgz" + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz" }, "tunnel-agent": { "version": "0.4.2", @@ -631,7 +631,7 @@ "dependencies": { "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.6 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -810,7 +810,7 @@ }, "type-is": { "version": "1.6.12", - "from": "type-is@>=1.6.10 <1.7.0", + "from": "type-is@>=1.6.6 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { @@ -820,7 +820,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.6 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -1061,9 +1061,9 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.2.tgz" }, "tough-cookie": { - "version": "2.2.1", + "version": "2.2.2", "from": "tough-cookie@>=0.12.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.1.tgz" + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz" }, "http-signature": { "version": "0.11.0", @@ -4170,6 +4170,7 @@ "turbo-cartocss": { "version": "0.4.0", "from": "turbo-cartocss@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/turbo-cartocss/-/turbo-cartocss-0.4.0.tgz", "dependencies": { "browser-request": { "version": "0.3.3", @@ -4676,9 +4677,9 @@ "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz" }, "tough-cookie": { - "version": "2.2.1", + "version": "2.2.2", "from": "tough-cookie@>=2.2.0 <2.3.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.1.tgz" + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz" }, "tunnel-agent": { "version": "0.4.2", From 3f41f19ab98feded843d6d6e83722cc75cfa54d9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 14 Mar 2016 11:50:52 +0100 Subject: [PATCH 26/80] Rename adapter --- lib/cartodb/controllers/map.js | 6 +++--- ...ysis_layers_adapter.js => analysis_mapconfig_adapter.js} | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) rename lib/cartodb/models/{mapconfig_analysis_layers_adapter.js => analysis_mapconfig_adapter.js} (91%) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index c314039a..baed016d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -16,7 +16,7 @@ var Datasource = windshaft.model.Datasource; var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); -var MapConfigAnalysisLayersAdapter = require('../models/mapconfig_analysis_layers_adapter'); +var AnalysisMapConfigAdapter = require('../models/analysis_mapconfig_adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter'); @@ -48,7 +48,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; - this.analysisLayersAdapter = new MapConfigAnalysisLayersAdapter(); + this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(); this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi); } @@ -154,7 +154,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { apiKey: req.params.api_key } }; - self.analysisLayersAdapter.getLayers(analysisConfiguration, requestMapConfig, this); + self.analysisMapConfigAdapter.getLayers(analysisConfiguration, requestMapConfig, this); }, function beforeLayergroupCreate(err, requestMapConfig) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js similarity index 91% rename from lib/cartodb/models/mapconfig_analysis_layers_adapter.js rename to lib/cartodb/models/analysis_mapconfig_adapter.js index de3ea242..99d34f5a 100644 --- a/lib/cartodb/models/mapconfig_analysis_layers_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -4,11 +4,11 @@ var camshaft = require('camshaft'); var dot = require('dot'); dot.templateSettings.strip = false; -function MapConfigAnalysisLayersAdapter(templateMaps) { +function AnalysisMapConfigAdapter(templateMaps) { this.templateMaps = templateMaps; } -module.exports = MapConfigAnalysisLayersAdapter; +module.exports = AnalysisMapConfigAdapter; var SKIP_COLUMNS = { 'the_geom': true, @@ -34,7 +34,7 @@ function shouldAdaptLayers(requestMapConfig) { Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; } -MapConfigAnalysisLayersAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, callback) { +AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, callback) { if (!shouldAdaptLayers(requestMapConfig)) { return callback(null, requestMapConfig); From a05f3d6ee93095c50d686fef12c75460398b5186 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 14 Mar 2016 16:06:25 +0100 Subject: [PATCH 27/80] Add cdb_analysis_catalog table and first test using source:id --- test/acceptance/analysis/analysis-layers.js | 107 ++++++++++++++++++++ test/support/prepare_db.sh | 3 + test/support/sql/windshaft.test.sql | 2 + 3 files changed, 112 insertions(+) create mode 100644 test/acceptance/analysis/analysis-layers.js diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js new file mode 100644 index 00000000..5d5c5a0e --- /dev/null +++ b/test/acceptance/analysis/analysis-layers.js @@ -0,0 +1,107 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); +var dot = require('dot'); + +describe('analysis-layers', function() { + + + var multitypeStyleTemplate = dot.template([ + "#points['mapnik::geometry_type'=1] {", + " marker-fill-opacity: {{=it._opacity}};", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: {{=it._opacity}};", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: {{=it._color}};", + " marker-allow-overlap: true;", + "}", + "#lines['mapnik::geometry_type'=2] {", + " line-color: {{=it._color}};", + " line-width: 2;", + " line-opacity: {{=it._opacity}};", + "}", + "#polygons['mapnik::geometry_type'=3] {", + " polygon-fill: {{=it._color}};", + " polygon-opacity: {{=it._opacity}};", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: {{=it._opacity}};", + "}" + ].join('\n')); + + + function cartocss(color, opacity) { + return multitypeStyleTemplate({ + _color: color || '#F11810', + _opacity: Number.isFinite(opacity) ? opacity : 1 + }); + } + + function mapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var DEFAULT_MULTITYPE_STYLE = cartocss(); + + var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 }; + + var useCases = [ + { + desc: 'basic source-id mapnik layer', + mapConfig: mapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.1.1", + "interactivity": [] + } + } + ], + {}, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + } + ] + ), + tile: { z: 0, x: 0, y: 0 } + } + ]; + + useCases.forEach(function(useCase, imageIdx) { + it('should implement use case: "' + useCase.desc + '"', function(done) { + + var testClient = new TestClient(useCase.mapConfig, 1234); + + var tile = useCase.tile || TILE_ANALYSIS_TABLES; + + testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { + assert.ok(!err, err); + + image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); + + assert.equal(image.width(), 256); + + testClient.drain(done); + }); + }); + }); +}); diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index f03d52de..928272fd 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -71,6 +71,9 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" + curl -L -s https://raw.githubusercontent.com/CartoDB/camshaft/master/test/fixtures/cdb_analysis_catalog.sql -o sql/cdb_analysis_catalog.sql + cat sql/cdb_analysis_catalog.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + cat sql/windshaft.test.sql sql/gadm4.sql | sed "s/:PUBLICUSER/${PUBLICUSER}/" | sed "s/:PUBLICPASS/${PUBLICPASS}/" | diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index cb8485b1..7ad07821 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -706,3 +706,5 @@ GRANT SELECT ON TABLE analysis_rent_listings TO :PUBLICUSER; -- PostgreSQL database dump complete -- +-- +GRANT SELECT, UPDATE, INSERT, DELETE ON cdb_analysis_catalog TO :TESTUSER; From 4924bcc2980acca027a1911ac081e4b7a7e818aa Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 14 Mar 2016 16:16:27 +0100 Subject: [PATCH 28/80] Validate image from analysis --- test/acceptance/analysis/analysis-layers.js | 12 +++++++----- .../analysis/basic-source-id-mapnik-layer.png | Bin 0 -> 28794 bytes 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 test/fixtures/analysis/basic-source-id-mapnik-layer.png diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index 5d5c5a0e..a0dd0ad9 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -6,6 +6,7 @@ var dot = require('dot'); describe('analysis-layers', function() { + var IMAGE_TOLERANCE_PER_MIL = 20; var multitypeStyleTemplate = dot.template([ "#points['mapnik::geometry_type'=1] {", @@ -86,7 +87,7 @@ describe('analysis-layers', function() { } ]; - useCases.forEach(function(useCase, imageIdx) { + useCases.forEach(function(useCase) { it('should implement use case: "' + useCase.desc + '"', function(done) { var testClient = new TestClient(useCase.mapConfig, 1234); @@ -96,11 +97,12 @@ describe('analysis-layers', function() { testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { assert.ok(!err, err); - image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); + var fixturePath = './test/fixtures/analysis/basic-source-id-mapnik-layer.png'; + assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) { + assert.ok(!err, err); - assert.equal(image.width(), 256); - - testClient.drain(done); + testClient.drain(done); + }); }); }); }); diff --git a/test/fixtures/analysis/basic-source-id-mapnik-layer.png b/test/fixtures/analysis/basic-source-id-mapnik-layer.png new file mode 100644 index 0000000000000000000000000000000000000000..bdb315a9fac10923b31763b060abc14528810761 GIT binary patch literal 28794 zcmb3*~V6z?aj7rbL-~X>So)v?b_VUuKi`(=Cl8g@VuJyV&)7E=O98^ zQ3?f-01*HHpvXu|r~&{G|3C-;JnX+g-?iKl0B{zSkq}k)%D(9Fi7wFs9-alv1iZU( z_!SdLBty_-OmgCr4gMq#GA12Oo%Grcq>+wD%x0$>r%Hf?#O8>A#f-A=bB(KLH!*ss)PvAwBR_s=v(_3L(!@pCWQPGn4ZYPp{ zEhTx^N=(dPAk_Yj1eE17_ez!q_R(nH)n)~elX+FTK--FePw+w$K} z?DKk@C4HZci!*HaNWgk~H0<5n`mba{hlNo|*kd!Q@8b-v3UNE&%ScP{eA8p9)BkIN z<*UiesN+?nA|!S5v6RIB-z6{-QC}S3zNY%pTRx3?Z~MKqAf-h31e8um-c>X~Cn=5iMVHJ7*)yE3y1A4w(nnpO-YDVyFGv%o7srIMQ#5la+ z=p2jDjR9t?noPgZ^AlqTbyRQfKU%an2P&>N9ntJQ9^CocgyMa!YSsCOLRed)mA!iG@3!w}@31uIrd3qEPn#hb}3B9BvNH zKaKhBpoEs>OE_&S;3jH{;Io#0xg~7tAK$=$Ek$lTPUbB{DPU&D@|`$h`~H!MkcnzW zI=jX8n3lwQ+bybhvzpeDO55DQT>IBZMv~5zrc#U3g4>ma8-=zu1 z;5Mg-m>Mi=m-2o2@2wbypq{UCJ8gC>iHKKWBojTYRVEU3oU<$CjT-`HlEsY}M3##g zgPPQJd%l9v@=P`XPj1I(0Z*XsFxR_q0e9OO>-JR1)^lbFDdD7s_YWRx#z7x&RYrp7 z^?)s(nY3?yF?RHK0)RgyGsf6%YW^Z)TBxL-DL;*@(l{|q`Cz@}iLdXkivH_k zo%Np50C4;7`B>>P2mS`oP~`d4yMOY>=AQ5%T*k~_uN{A6r-)TlEn!QHN3Dq5z>G^i zc+>?5OB>e>}c7g#&nDl_g~ljVP2jJrj&3%hpd?^n(%e+K^jTG`H& z1X{Cr6fO(xKA0VRBth|F*F+KC_TeDxk)QD@m@X zJY3wSe7ENg5VXp3aO6_10%*zplhF4Xl`=^B`a4CO^t~o!FyM9cwT^2IKb34MN1f>64|F1Nv_MMtxja1=<`u1S+j-?x}Q3orv z@k%^~NdA|HvtoelmfLwHqn0KrssGID)=dWakMnz!p{=q)o&!zVnb>-ki=d;l&DXG2 zNbLjiy=&5qPtTd@de*)>hx4MIma`!M#zIti%P!0`8S{b_>3c`s653ZlDJAoYfwhSP zL!k6(5L7;j_C>8sRvj13zK0j8Z`fwQgo51n%!U%rk6o}_+-0AC&()JWI{k{d&dqEp zGoO*iQ&L0Y?QvqtFV|F5$$`dYlI@PVQ+C2BmG^jTA(t<|IQpj~F~r0$kk12 zT4H-sU+wZTA>3IN^o47Ds+11RFT%f1m%O=QbiKAzXW?Cn=KVCo<7wumxf~{X{ ziJ{I=3>j^a^M<#8%7QRXAMfX_t#F@BaPV-ohDCCm4)qA+gwdmYxh|Hb$LV$~$J1Dp z^m3L2s}DseB1m^8tsdiQhQ!#AAlG{%E5M(ocE$bZpJYiEV&)Q^;shgR+hUsyUof`UiDDN0>clSSqAZ*qFqw~3s4+&uC32#6LSQJ1zzY0|f zzmU`5XPf%rf75>#sKq_4R}BSs2WrsLbIV&N|4C7K&(g2`xuSX$ktqAmdKoSqECjp| zBNXDF<&hXK#N;hJlx6o5^QT~gw~j!)zH?shybVZ`<--8MYK4Rb{|G=>Z|FIgv7@85ncT^%lLwC zy zpv|u5&ip|af8$Tj$Kck-PPYUuu1NL-%zrti5YH@`Dc&?C%n8!AqO^o&Evsj#+Y~oz zOhRZVTG>B&H@parEi#-FX`CjJ(z#u;&SX+8Y~+LD;dmbVdlzmYk!8L${?Gv>schSk zk0^xW(+|&Q%18?kuQ*qZNiD59ZQRsXAN0N9QxC5i5mk$mzJ6#tXgi}Y&n#DakbK*B znldja}TKj*VN`hUQ-+*11B4flIua$N)I zq6su1eujz-$no4K2opRmXX0;Eu)Mt=D7BH0fI?kibW@@-`KTIJJ6HGd;g=-VC~a=d z+Ag#Y-FW*MIxGYqs}eLd(|-yyc+NPKdE6bz!-fC=;Ej&VH(cGf2%xT3MA7nyOx_gi z&YR-HpOfpY9y_JAJ>?cx3L22o(?pseY`v626&RQnDs$?DUV;-Op8t#7@> z_XST9xzrDDlUpg%r|uZfTyAch%LFRuw2ZAX!h_QH+@gO@WeUk#v#7IG`T!;0rSpZtPvY<8EKEk%j7wQss~dUxc+>pTu_SUFsXHSuBb3Ra(lZ|iSH4G1njS{^57T~c4ew23rEc0h*42cd zs2Pdz#cn#GJ!9RYo%|gLj#@10DgP<*cJQqlEb4H+uc#^{kx*in7u@L8D!twMt3JW- zc0>rNSf1KAuCwse!`rLJ1YW|rTe>IdKTpv4WX$UGm30Rdhvl*H__8fvC$Jv->Qf^q z$F}~F9;QFIO<%v7Cn~hn&UP^_z&*UGv|7#8BWwuohk~n4=*$erwHRs;_HbZ)6%EZ2 zGQT|4*!@muNVb2&pZ$_FuV{Z7>d9|lC9W+VOYhjkyu~60QE4Sa^$*UV&yHwb`&Tdj zsx~c_qx1Jbv2YaNDi}%PMFx%Dp;8UT%h{;rLCT~Cgv*Lw)T>7v-}sBNoDT?bVqB4& zSxpa;JeUbHnLDCI8ZOP_mDk>uo4*;G>0H)Qdl{i4{X!?X7J$j5tD^%WvPgSWG3SyZ!WTI z0{cKNP9pljV@j1(-IX9NF?c4#{B##_mHd(qZitSXEncjZRpdCB*G81zj4a zsmAE}l$Ock(dIOQ8F>XQnLpdZ3Vw!vk>qfVyrTUEat8!gcD9Z8!)WLm$$xY?%nue6 z8$4z!FW;IzcbtO7X-V3z_|Zq>-F8~&Jg5m&4%;tJ`oIvy*RefR97dh6xlK%7=ili| zBrCMf+%GugEIdyI&;qX{E1ok|yj3}Hjac7asLz4V`FWo*!)L$pZlIx6++U~!-J8Ly z!*oHK2xwmM9W*p5rBPhB^n%zV@L8ZNYk&X0RMtA2>%}N!1huOYHA4p8@Dqrlx*(bj zQbf~eZ0JyrM9`r}b&mdudr!^PL8sEA$nHTirk+hsT&;y9$?z!b;hN4%(}}@tC4GM8 z!A*f+ug?lvvygccOvc6wa$YFKN`V6S8%y$+R`$BLWdrW*RJBm-mC%!kK+2#0VhJ}at@2RZxv?`>&{ z)5^Mor+(DbF1}9&Gd$}gxo6U-fp&~;iuy=LQCDD`9PHVD_Q%~RC=vf;(1R>#B>pEg zf3BgnUwGT4b_I3dn6GP6EwZyoE4C^@#;+sxa5DqZ*)x}fh27W=e%bhXbfRDVIP|>` zPJ$y_rF`e7*O|)ijr$lDLQv7QfwR_CZZeXBoji=-2XpPG!+V@Fb4zCrOU9Ne!qNCU=L& zz0YC7{sdEAa34_=(;bg=n^`q=7FlI8T_sy3J*Kg}X}kK=QJv~W(kBBzecoRQ>gE^i zTxqDFJa>onCb|`h9U+g|g|m*_ciwU_RUoP%DY4f;Jri<<2}kK?x}NFvsaXk=emWMK zFi$(NJ`YGsmxaZwm@qZPQqBBQ7=ehxq~fQF#B^|LZ`RY4{aAC*wZ85Z{%*m@;AkTR z*to#9wu(#2yWTuc%g;+0AY<25AcZ4>SfzO2+=-<^n3}qW*6pNoF=CevAiCpNg%S`J ztaD%8w=MgslqZDiY~8%2*$i+=(amWwMMIvU_S0lsACkNFq^T>+^u)Xd8`44<0S)$E zPTyn@%be%N`GJTDQ%`(!7;7RCwt487(O%5Hse{_n*~_u)iKs%6SaD;BMARZ061n>z z|9}j^Oa1iPI8_1d5LsplEx@NCeF>RJhWG6BD#M}dWhHyfrQHhd5G3An;PTG38-z7S zvt0@q<+owS^mJDe@9VW?@-~l(;oZt9u6|5&5j{;QkB$AcMFAwgEnxy;eLD!Le{zb&dnnqM=UybpBB#@CJm^_AC9Mx+042@f(lLZGzgL(1=cJ-o zQqn!28IFpg^^MVh=$Bqu#nTXZ_G=EBCgPhdoZOBm#Hfcx1nHw)2CgkJBt1=HWXei` zefB%eZ$lE@udLM`3zw4Z#2xI@`WsJdRZjYdr&Mg?HTrTZ1<1#SkAN*z!&AZXbHA-O zV{Q65TByi^^Z~Vf+|i$O840M6)S`Ak|GG)*65k-0n$)jd5R4lU6TIcTdCg??w*+L@ z1awoZXdpGkhIF2w+t|G5FIAN~nW(^TRw~H{QZ5#nqgkps`r0__oia=w`R+9`v4uHH z9+Eu4IAP3$?nEfkTJ~B*;Y(REnLC#|s9Q}wvV?oQ@hG&YLyQ%n#I4BUM&f(|1tDmr zq9TnoFfcj}f*>nYs?>em!PjKaj9umHKgVNfy20FJ_@F@J5be5%L2T&iZxq$m@wptF zF2Z<2By>vAWK=@QcYUW^h0PNB*0M4)Ou@|1pe(bIkWpM^ z&@%mUHs>xu{r%vMRDVicMmIZDbw?G;@JU-p2_~?PtiJ)BI_VP z6q)Y*ZY%&klL0w1QK{1@Bd4hZXA!MvZT!1Yd z+JBUGy1t0-0MKRE-z8I3b@}XKRk+-XlX)Iyoe3jQ7J?_B%Be%8R5)ak7@xadKld(PcZcv z$wbA;ILUic%wmnFi?b-%h5sgN&%ciG!9*%(z;#Xs7B$M~^d-q7qn_cE)Xs~~3l<+u zl@iX{rhsE|DGP{%ng81{;bOb#d?eUrOf+LJ=x>!QPpDCNX-h5GeW98;aT9-!p-t^( zHGxSC#uyR45Uy3SXp`10Tzn3oe=L^X4=wS*-e&AFd=x(8&D@fOg$SXp=`;O7L3=Yc z=m}v@Fu~aFZMm&+X_5ToS}!`mxjQ|r#oc|TvjJ&r{ds}>&5mHV1YkD@TE zBW_>1FA4ar57%;5{JS4T1C9J(r^ zDY;)4&qO*OW3G&riaXyn;bisCM)2yA$Wj+~ln7-bc0`6xG7|pudPqHxor+kQaXRrMotX<0AD?V~K+3> zaxF9?e5X6wts(sxwFTK=Vl>2?%9x0(+qU++(6)~@i^sf=I z#xW+=t|EU))d|uSs1xbsdMg2qyt)l23aF`tGF530Q11C6z<$8H^1sp zUi;Jc>z&)yME)ce-veX#H=~Ac2cby3si>*>#X8iYwFeqnLeO8iB*jUw3G|12(C7$E zJ-eQNe=ZdAXAJiJ?7yZBco#sQ6xgIz6NaU6k*@{A<>BhsI6>|X^39nxVEDk$$A~h?{4>SyD;iPzfxOfn@*^zkYpq5`KU#MQ%bAAan%c-t!35UK0ntyzgt z&6>3xM<#Frdyv0#IT~n-61NAsCBi&GHNTCabWx%?Sfp3H4l>F}EiId6 zA~y6T(xLeFFFwGS&E(f-pHI@|XX0yciF6t=`2y>~rLjA-nCh-6K<5*a@yLBmGSl66 zb`=17M6q;z%cTpAfb`as32AraK2BiKwbcnL4Bz5KgKhxd9l>V7U}r|(ProjpLwmsq ziHfW$Am(9s>02x`v<-$Ld||EtSAnvbkqc{csf}Msa+lSNFhdw zp$_@U^lw$*Nh@ zS$It*cH-p7VMVbKJm_?(KdzU*T7p7FVZlXX5)maW#rqgJv&t_u8^S$czx#lV)?{{Y z1R4~&d;D3ck}!*$v+^cXCZNG zm_bM3LkJkYA2-&vc+KPpZ@YX#-o@*FYzFnt)y)$cBva5j0M`;^CF@KVahomvqg+hc zNAbU43%ZXDV2J3>qJZQrotNo7BeV(H7KogRs65mR3m5%cN;xqqVG5tS_jatwuq$Sc z((lq~?=mZ90anOM(sJ9UvEE(8#)b$Dmm-(7=ic`greK4~A6hYRLNLFF1X;u~A#hT^ zx9^VI7o{T9sXVa?A`kZ&^*JJ1z&L)5S>yp-@)dvMx6d&RdJx&op*HKv)WG`MR zS5()l=%)Vs!P0YvAFbbHW=er6XQ(*@HFuokaf*tgFi4qo6uCl zrUP)d?~_rG(h0*Ycr2NkpFVtPswa>7tdVl{sF$gMja8)hz0L(=)#e7^%!2{}6=dx^5_C9TTY zTp+@6A~NA!1}X?>da?gm2sP1=17M*fl*odo_^E$6TlhBwo>4T8Zb(YMqfZaCB(TRXKIb zqEe-?fj>p++=mde6j_^PR7S~|7U%?a^iRa;9TRaufJO4i+PQUIL=ZY2qMB=5sKj@< zvq7`E@OA8&Adex!svfBViV+L}$Yw856U&JzzmVX_R{bexYdSIw*upa98)vfLt?h)?bG3)p;&Efuweq z*bOH5wz7T4HYb{()d@_EX{CcRqayqrQ^M@Vs|sVdHW>kL=_{}>oM)+7rd` %aK^ zr89Y?PFrd?Q{PC@5dXoCUQFJ+kuIhMOb*bmozfB)rUR6|O3++av= z6KYp2KBWudq!zU{PW3vU=Tv;yNiyqC3MaT*gBU$Bjss-%@)7;PT_}MjzhA6uKaFo$!N6={q1o(9hsCz+h?b0 z5m;$Dm+g|_)Ku{Z^wE3?>mB6mHg6<0Pa`V0pBw)%`{E;I`AJ2@%VV{znk)^cGZ~LB z#Z_^X0E#Bd)}bR(nDjmwV8f9tIq?6g(~!H8i@W%$ zPO_Bsg)vn#Pqw*OHE)mdvZ>u_J6<_=JOWLV;Zno97~L_5X!FxvsmCIf2p#@7zJo_u z#Z5-wt#2p@?PVY0L%XBEwHRE4ez!;Bxjtgano>ZYh7AgTG!U@Uvp=w9qtK_%kUp$U z^`h~%7f){=1@>ld$@}%C8$Z7SFqNqwjNMs zqmjk6y{i_?$E&UAvua32C5kL_K0|?8B1gL?hcf~q)rMAz6U{d0a=2_IANamj+*uxu zIZe0%lG%T=O(IHX9J}MyL%RN9i|!Lj`a@;Z^Ej7^TY!?l#s3%hofd95aMGz<5>1uW zk9%fJ;L)CIwCOIt5{zb^nld&038=V00 zv~|+Vo$4V;a`2c?89q23E>KfIPds2`m0kZKe{PxQJ1L}F@j=Z1P;DT(Fnk2sDV|1y z-5CECLWg&9Yb~>d#xPh}(_0yX-!_Q1`b0FQBc%P!OW^Q>@~}#1=tsRmm`1q$t*eag z$!d;*t~X_S;TbRY^$;c6M__UK>Muy%GOtWm+r`At1(rlPngRELiHj*&%lZF|Y~pa< z+f}@$uu>>I$GfY?29ay3IJpePb9KZ!z^pvK|DdhtTsrB>J^fW81}lp(OSNG`=l#~& zm+#v?=Y&dhxsrL|=tiB6zyvmooZ6KGbwU7h9{gGm_E~tyj2iSzn^Gru{&9cf6J)|~ zu1~qA{GR-oj>uOem1>ic2u+RtptK?pIYi8v3HSV%^&Z;P2JtfS?&{5K^51hp1;xa2 zJM0FE9c~8eE6j#900vMcAb-}8pAj5z_eteRC7!Lfkrve%(?v9<2R&&u!Rfpr_TVl{ z*ZSS~SeP+F=LE_q*FwREwrBA@5Hw#eOSm1;si_DJgQaSJKD)cJLX?hIREA};U2^ZI z`cMC{D#Ijv$tkXv6ByAB=5j(g8g20j6$&9eaRx(K2TthL95vTQDbUs9cpsiGu_(5` zuDZDw+!9bXFm|LvcwWg17tp2@khv|fhw>uL4A$~@FjnGw6+!s1TzTdnAi`+=$M|yg zlZsH**dBXLd4WYIve4%TnoP>7!rN;xD0&xF?VC$mLVgp@g=2g?f61Z;_L`yhN`s@} z&<~jY-=6E%xK91TZzjKYWg{iB@M`$z`xHb-=NN_PqOS>Hx23eL1-q{`!6;D=zuph7 zQaqD3s}f`3Bqbzuvno>J?d5;rNYm59DO;S09{%v1p@Qb2@fD0@Q}YG-ZRy83>Z&9n zcs@PUsle(X3YTt~0{1_|s&u?1i>Uva;=!#V(E9ta+EOzw>pAj||0ns`&x=5R-@Kwc z3BDj4qU75Em4f?1D7e2mE^XiAJ|OME77K;#@Se z!yBp8^utu7=9i+gaKk+H6g7py&dE+JwaJMLD##w*;iMnL%jaL??H4HvxVc?KjqmisHYDVZh7 zkcWmXhrtI+b8B83XT#MGZv&W39SglN3Oi_K2#&^?{NuVN;wnmV#9e;J@J-0Wg9%0< zaJZ?SWNGQMSda(1mFeN?+kePv@?T@H3%+}q%!yNkP#DV5(LK(^!%Em}ausKC>_WgZpxUA1pe1jl)jc!sR4DtJG)HTx zP~-^6EpPALRudL!@+B`Wu<+M5^E#-^%x#~CI=;iN4_sI~zZbauc0*Zuy9!E(ar)V8aM?UdgM+P^-LO z_R!Ia*<5nGY23OaTm>?Pxb^*N9isaepcz^-)%>1AVp25!OEmH1X`f56R*VL|N&WDS zU5tuMy+Qci@J%DAx|$%HddVfrBQMcRzO}>_*(bAam~z>km5xxXN3V9n;{nZ|y5Ng0 z6M%^$KTkA5bMvQRN%+8gRC*K%MQqBgV+RsAAJEUn8V|$hm`1Rr5@;XiP^i1(*=LTW zA;-T|4Zm8sQN3J-tIf8f=@94((gY`e4CO8Vo>Tf`&mT70TKW`jSBL+BTIN!#^}TaM zn>OtR7T%$w0BKV4rKnQ$QD_;zZeZtk>GL<{njWYg64{#iY}b=DqhvA4+^H?vdUTN- z_?r=Qr`R+-kPnHae?C8wD++3fcpA_k=mlSSOSDr;XWJMlksM*HL!jth06S6A&=j7V z_5)A6szoo!U{f703{%mtLw2!ky_w^Wz@nA7Z5&R)p#J!OgeigWA^*iM(o3V}5G4}k zImXH`(Ha=?SPca4weI1^E4A;l_dLmv)9b^Ra2TVBMeHj`S;QKjx@p_*Ztv!2)vIN8 zJZHhvKr$7%E@|_!%l??=BRrMwEs!9tZQ1raAk+q6Fz~3*R51i%{G_o%r0&`~po7bd znx<^HgRaW#3LalUFd;wFUn>xe`gKJBA?=-mCLC$WnI7zVaZwSuQ#Q|)tG0;Z6N`wg z4UfElN5+s?E}qz|EEX2ek3A_)da+Xc9%vK=f)Wz#)GsQwW~lR>u`%p-;9n#<^#0O@ zW4^vhLgZ9%DV|C+MaPhJEojYeAGz69Kck%!GkEwH?s01rXLo~eHhbI01-xrQ#Mu;X_E$Hml=9- zxi8h1?OFx~h4n*hZ+{V(B(M_I+*ok94GTFFRx|1bW>2^QITJe!h zE-&|=V^$5+D!#SC&nw^Po(P&Rk;N`Ora5l~cPT!49R`Rrq74>`c!fZxbpvjJ32U)L zBof4;^F8Gib27X#RJ(5!<*6j(W%N_vvtRse`cH^#OW0On`EOThR246VrGgn&i3e*$ z1i+AVY}9DUdeLDw2hkjxM;q#`tU}`k@^v4%MN_tlN3YLwr&F(HSlH2;!k@G7zko1~DQ*Z(5K725~2| ziWQevm}L7wijmh&G2)v&XMDC1W=$JPS|Pmo&7#p@G1gxAm3oZ6yZj*+klrO|UR$5~ zMGUEJSJX*j&@4x77NT?!ZzF`S`vgLReP(XV+*5c|4!NH&O~&NwUr|+>Z1hlOosG9Y zq~;tHfQ(@y4O{&%HP!9_QOXeH)R7fzrggeYQ*4cbbF)M2eKM~9-`~`88TW`0x4A!o zbWkOcuHb=G%Qa>>YpuH9Ryd$Cb!SBCg`jUh8)d=A;*bAb3N2g_N?$z#m2%oOQL{`2SwJ=q+ zsU+MV4jIom5lAJ;u*_9#T#wSGL4uLWP^QV;$G9$>RuA?MgOHmEq9Ap~)c+`BUT$b>54iUb-z#shRDTmHJ+Btl~PQs`2Vi`6#g8lmeIT%_Ul^(i(;~)3W2zp4I%Y zcIruo2Xwg6TPbFi(9L(dqPnHNRiwA@4umH^l~$db(#hpiHmDmV5uChBZu`7M8e$#h zJbl1B7k=L$t>ruG(}5W)GIO0lc3xhPrz;Ig{15SyPybY zb<+e{PdU-;(CxnlXBzQ`Etz1)q0V3g^mT5a0{bAZ9h`S#L0I;maXP@0WcsEp>@e;N zy5Ny?F=9D~YM6W2I%rkhTrJWLnMFkAyDRR?!|GlIMvSgYv(N%BEy$1N&yR-;GcO

QX>h;-&%XXrr?T zpE-FPQ|E}HgOWSmE9#liy1&Gpr+KL4#+$3Mx2l1@)RN3Gi>ic33-#TlH`Sfx3H1EDUL-+yZ@F z)Fb%GWopBoJU)Q~4N{?3;t>%ee9`cV*6f%S5*J2?8zBwEu_J6aPxvEO80`MCc`_a? zi{Q-Lreb&%Fz}?a8x)9^HlxWjN*mO(hS})4KbGgx^y_BH*gV&>@m;6X@P-(HH?0jU z&qi1sZnVL{<~((?ea%5(qdrJNCo=}Xx zJdgIic=d3W(|T#C^S5BNvT!3@PohOANiQm}uKeKNyRHf~xx7n%CAR3WUU$h@G5k%d zQ#I-@0YPUtqwlMCMUV8Kax}>=Hg&i{AGUxP5v%X86w53E6G0+WkLU4&hzHgzO}{>T zuP6w>T>8#h3YlbyF}%wL%vcUdHc*H#c<@Y|$Izl7-{aV)x;148z{^JKE(AZ zPo(n0wh$T2;z(&F`=2nk6Co+uhY$KvTyhhkEk43Twza}sIyJI|3Rglv8Zk9;_-2c> z&8n6GH^kjoa?2+ZlM5-+Gnu>rLJ%xDb^mPze+aEQe*G&%yNV$U+Z6aHvz1 zx3U*b42LSsk5j{rAZ4}@cRaa*hbY!UWDNaW#*;(6af<_5Q~%OR4%CcxSb?}}OJms?k zE`;H0VA4)B*3kCHTkFCTtGbpV1Y3=ViT`%Y)^o1cJ1v}=xT+p_`{2s&smp8~C6_>; zeV{I9vX)!2A3<5mz&50kB>(i6@k*c7@#0H2{Lf+lK~25?#pG2l55AU><=9l|{wI!t z%k;!Aj!+9aNj;;k^%tvy6^rw*9~i(gGVplb78@R{;>(-;&cxYD$WipvYMqwTPcrNn zzqx9P}OQRGNaEY>xgkWcWio2Q9R3DyK_pg2nr6$ z0DNa;^E)WsK)V*2WTXlE&OMnW5TjRcTJPI>0nHt9dgz!Y|0vtgTFiT3%_}w*rlELX zVRQ;>(BYzF98)_g9k`*yx+q0JYmufkUITe%NY+sH+dAY>fjj@ao(Q}N_{#e4cv9e{ zDU+Vd$2oDU0>)p6UYc=b0G8qi47#!+ysKms4HmIiA%R;g z5HN6N^=)vu@YT6Ku%O2vZkdL4G$6V~^RQi&0FzdPDuU;o^Kh#Lvy>x-7#xIl#i*q; z3}WS@AK`cmfL^4q)1i+^SzaOZH>7YPgR(gwn}=x8@Ccqlkff(lcBf4GWx;u}fv1B@ z9XEK}MI{46lh;OT0W*}2X)<1&RX|q4!Po=MAgAR8Z9qLfF`O$={S{{;b16?W5mAqp z@!o9sW~YXDV>dOxQC2Wi%Gd*&R=0?dP9mhq9Eb6dZx_+i@Z#bqD5(N5L^ck>}3W=BDUo?Xj7{B3nizeg_Tp9rBf1u%y3^iMs z=A%9yR~EWNV9r3ceMDjXby9kAIjXQCL8C6~W_nndvgw|K~8YqY* z2)7H!iVzjvUC-^F$WcC`}eIhK9w|JT7fI zEYHZRZgN(Kkigd(6J2@drE0%uKSMe1Ma#KGteGcBN8cW^F>0re@TtR35rv4Co=nb^ z;Vy=kT=iVPt>F>O3=D|_NQub&O^4qe0ro+>VB8|)KL6Lba^K62deKHej-WlVsq37d ztu2TJDh)uRPLf$!oFz48=8sFW)vYGNPI9998}a+}U&=(%k)G>c4ppIrH(=tb?|Imj zVfG}P2m4>GI;&#~7ds&NI9`pBjcJ(L$yXCH>-C4pPi^ffSYvdle!nTV_@pN9#u~gh zGt*{Tx)tnZSxuVB@){uW*jNB9nQCg8HT5$`T0dS8VpFpfHUk>EtR$Tx-6SHWf}qjB z@ATN@JsNe*;ubALWB*p8$%&NupXl3ol6pRUpqAxJ4eQnBSi6?E)?{l5_9@0N6G{5w zW@LN!G!1#w!Wc+gqh@o(maJtGF~&#EYf7)3jr`)lzORIj_zVu=rtonkAHjZq+L`dH z67Ki0BbG1nO<3ox0$C9d-lBoyu1$;)lrJV$F7myK0O3M_?$b~KI3-d5F|C#Y#0t<2 zHtb%OBBM^Un6-oA=Fm<)QKfV0+ze)6X-kC-0UrJy=L{V}axbmN+qeDK(G~qsysxL4 zPLK|PgtkeU{_h8$oM=E2t$Ly@HV=fHVI)$+tO?^pQ%>16U#7}6S7&X%+_%VmIqpk! zz8lzrqE`4;b^fseF#|_w@3S<^S>F3u4BadPb!Rj|8chz+KnUM2 zaYW6_Qh+l=BVWnSE2cvb9cs?9=o$@E;@K&2CadmBrjh5M)D9=`r!exQZ59|>Sclsc zh95wA(qGj@G!>Ik z&u#e%Q^nsv@&bIP#^i7zdL=i)D#+3coUtX(7#1h&C#Jd`z+7o&CY)_CY}y>#Rp~+v zZS6tlR$NjdU&i0RzHXl?!Tx$h<_hw{ZP}&>wulJCH=0=ENuUO|Y~H-7Iy{9&s$Hmfg;GGU!B z<9iL)%d-_heZS*=TP^+s3zQHYWXUGpJ(jRFS+ltBnA@d)w_uSa@r9t(+sEgzBNy$* z5#hX0y{vz_$>(n4Kw(!804d|>JA>JL&g&y7h0hAOCmiosCEZf%l$LwjwVu-HeqVVM z?qY(Ctr2hEMvh0J=rkGa(0=RU*?ikU@JeiAJKj_&!QPM>5>^DRIV!vqAP_cnRyT5WJhMw-l{;6csq(&5pGw(3PkUnfs>n} z3*q01OnbUoKD6`?+(bOmlMeaQfxF<&iB-sTi=@6;!*8jb>PL2Wh<8W(_Y704J|WM= zv>rb-TNe40%O`Ba>l0pcppLU&3@C2b7*&~C@WW-K8cxyjcmHPbEKsUvG>zir?MHp2#m@Xq>hRU zDqivxQyM8+9l1tD0AlP)x?7Fx&dvIGI7r0vuC6SRT%9;5H#j=Ta1G6^Cw{8CF&}}1 z!uhX;Z>QulE8P>)n1rgCvNOw(y(f<9{1nim`(M za}=o=JwCOTA-@z)pml9}%)b4L)A^|^#K^m0vB_$)Ccb#0bsW}V4X6%Kiib$5! z;OF3EX)td)@TX=Jz?+0W}DNf@n(bSB7C`6Be=L`Z-8ULE4bc}l1Vbe#Dw~RZ54&Vw!rO8d~ z9)tQuKj1n13Yr-ZlzNq@r+X|VxsidMWMM&4_C?tl-@?{RVL!gPKbS&dnOI05Sivlxdp-6-$fwQoFV z7U_x45{XMCV6HtG4Hh3&Eg62o>e5D@>e~`2Bp!%7|89HYPZXJx)qhp6o8y_PZnY(y zSL?;e_ylbUY1bCfV-!zQGDsjozB7|_@gkH=0DCfiv;C_||$!DEO&XpY88>2sRdpvi=$Ju&ftK*NATc#qj zC|`oTH+L1DiS(mkNW%Vu`8H8<&7=9^$B}OY6%~@twb1)nN|SiZ_ekOII~k`8hWv=J zvKCw3@0}0rh(jCmiF?FaXnus~x~kGrrP>)8gt`hi!6Guz+}@!oQrO?E`&VK@Mds7? zdNOxip09ltFFyZ#QCSq$*`M@KADs9E%+VolD?{Phr6%|xbh?o>nj937Ye|O2LhRzA z`C}l8XE`eD*!GYg4YancH=nCINV92_$a-89MjlmzNN(ay$! zLXxVBkbaNf+OEQ}+d;sC1Q@F`o-(seRfyv7W`QRK?W5}f7@@DZ0a9A9`GrOe=mAPeF zq@LstOBMA0*abQIg^TK%j>7t7dLI0pn@iwft#ADp10&$`l#tE3W5Af(S4W@XX!*c& zmVRhuHygFl3s=n7SvfLAI-;%+e z@ZN#E>2AF=qwu$UMLMZP@0&0QjQ-S_4EEa1;dlK(T>l=J@*;Q-1hwa!cm$}W1v-}v? z4i)1qf`+KNlGNU)&cL-&_hWE^KR%Q6J!eagxa`?vsI`<8=z&z02RMZ$+RRG_Hhn)n zlq_Si1o+(HRd-+vL#T%?i@|3bz$7D@J0BO+3@dbF_r4ys=<)Fj>Dcxf^LE_yX2-Ay zN-V8VgX2|A!m-_oV@_9WbJ%>mOC-_<9eCB?pP<5d{AUDp2@$9i7k!=&<~cxb^7!-2 z=GiIR_Nlq?^A;KR;`eC`e46(44z?$Yd))P!L?<_KTftHwc|q%~F&LCXgijXYuI>%^ z?rTdY=1`f=NFj8YV#-)xe25uRl75#WyM4NhF856@0F;hiycMcLGMGH{C#}24p%BN7ca?1W9z#He|@X!Phs2sP3al` z)BFSxH1|~`#X0}P`rqw_iMlEJkkFQad~fl1Pn*h^vadTU_@1Wh`I=^f7_zla2@>3FmWo0+@u}&K)xV z0}Tk4y~<=8HU-=@_gi#6znY3Yy0W|1TCe_WX>P-X=#w*Qed6w!Xkjud+il-`cw zLz-f6_{4PzH=&EV1IJx&|BiIh7_9x@xasiS1R274cfU4L=)vmTQ>-&)>rto`f2uKI zj*4g8J|jf8DYA5^1~pi6qUb!UdX|$GBVRthOyJcg^sT5mTq5*m{HrpnM4sg?kX!01 z#3bOfMRhESo?%&UZ2N0+eVPBIITQT6T=M&YZn0s?S(JE=Fcb4pNzWg2_|kxBdj&to zodlS3G&NV(%ohBHt6S2nmPHQ{>jUt`alPaAo#Q=aXoO7VgeiKX(!LTMkDXdVVpzIu zE;{CeoJ1^TF+AIAsY8Nvy8_?emVhEZTG1rxvGO}+(Umm9wJ)8EBezrr=})BeBw@iSr#jLGTN_l z>(+vJf2WfaVarC}P==XPGp;w}gWbJ+%QgGedCyw?r2l!fNC0LyLOx%E+zRQY5JpdRW;LEu$T zB2Eiphh&`}uAO}+P6wY_NG$Is9{BcRNsJ#Mg4-8v<|}N7%2}1daJ+pPv9vu~F04Da zf%E0Bq4N{BLcm6bC$L7}Pgr>?0WRctNSHrZ8i5|7axxh`B3pL4QmG4RinbFo4&1dc zAx2#N_qhx%R)LCjrOZ$?t(BVM##Pf&nJmB>=718?s4*=`$(EM4#1;jv=qb)CC@=;y zmS&kg7?s)O-(S&kFX*)H!`8d&qyscf;=G#g>NbafTfsXAv`1X`EsC?#+g;sQAJKg5 z^jX_PRhQXq!j{7VfMlY+rhd6)JQhu&`XM56hBR69zbyh(?ESp~lFMN@9~8SVIsT`Y zjfOXssbxEwJ=o7Zg^-k`)l%?0#c^u9a$QzVU#L?rX3`VEVrACzA>{FGoWb?T)D%sQ z5*1hzz{&yF;$3RrL!V<)tpc7g#^_K)%vO781*GREZPq%H5h;>|qU zFx07gMXd#Bw`{qC>L#Rr;L1bT7ub6J`Nqam+pwg*@W+OE!lQe6a+$)Aow4?g;QCR6fClI=O2 z6YwUC4y^;3Kq!KwJlvilyN;-zHiSB|)GpXA?vT|l#3C-W8Ci)%)Rrm|twDJ&20m+&^|47>?{Ty4DRr3wQ zaR)n)e3Z*iSi#B0j{IG?9;Vd0#uzENNhx9iTnSpKA~rs%1%t;Vwic(7p44yJUiu@%hwoeNC8Wdt>91 zz3}4d#JlGw>Bi!}cyx|(FSGpLYSYOd5dWjTm2iGSZ?+(O=6mJFD0Li^blKYi4VwdY z0Cl%xl^8PWMtj&nk;p|Uns>OGy^HJWXN6?TX12AX;!+3^R9YYci?_h-wZQ_1Ow`43 zS`Gj7dhU|p-S*gJDFc(uP4|{AGSK!==lqy|ZWzi(P}LiCu#uI|$upi|oYnPR3cn@; z!n{hY)0_8cbDV+NM~_{55= zw0>^oN}=oJV%IJI%142EZCC6x=;bq-eU*YPUr`|98XBQVv3`sAl4(Bi4Q{5}{z;zg z$6+$RKs`fE{Jg*As4j`9L;O^B>(ypl5Dm3ZNM{6g@gye_fZHZ#emsDmr>y*8&L?)z zEq{k(*!o6OQWNJZILUY!9f)cpV6GDtn9&Z#Qj>J<264u_z=tAmCt1u<3Uu^Y2-K{| zUt34Y<6w^y4JD#vMVY${#ryik-qF|{;cj@v>3WSGuA4P27Fc}kt8YNlCvc}-FAr1h zVqPqGm1^%!efw7nDDp&j-#L!S##MNBLareDA>}DW{&iiCKxR1guq0|0C6Irxo z5n}f}{FKHMvDRKP1jqYK(2YK;WQ zZq{u={n5^er*f3(Th!U9cxqX-jIktND^wJ-^*qIFSr1w9&i`{%0%G?&qOM zW+QD)l=@!sQs9TklyoRfdG@pr${of!tHCW%Z2RNQc{U%46YYS2HMi|#5BKauexjv% z(cc&1IEpZ9eX8CPP?Qp&151i~=$U5@kA34^eN=+3#@I()_FfpjI+Z1iXF)*+9 zDR+q9!f*RT>1z%?p@1+8mn52rJqt?`ho^Rji0#TabS5jpJs;wBBXL%eCb+`o6ndefOPjfu=?g2;O0 zyj1L_;*fauRvE@aj&&!2#b^80CR41;7R2&gEAcFzam5{Fvc{Wb&O12no8!MdF1R{n zd0$F!eWA3ME|od~zx3CLpHF}KJ$<(2I9CV>q5i0@9EG<{_SG=>IrH#HiVg^*+P7X9@P*i_)Bx`y7`3Jzf3(-(Dk<|K}|ct^lN|^vAdr@VoW_ zas7hL96X1Ws(hsfCXt=AIr~uSpW+8%`2-6jlX2E#$!jKJY@$@QV-d>%YfqY|D1sT8He*a}ds!Ph&Z5vP^Mn zm!or_2kyRb3rzt5bko+_=IAP`G z^#b(OwQN_sV);Kq6+TP`X=~Qd9&Aw7&5KfuUodwyof&1BF)5SD@PU>-1ai5hq`QfM zA;=IGa7d=xBSqi%xRKd4Xu{ls;Y)jVa3V?CF=!<<`sTv;OI3KnbQ77pNXe084AY9} zg0%{bqh-d&$JTwO3@JL`zmLlNP5VqAFc$D$!&6JO2~9|zQVtQl$lvqNCQWouAyw=v zsIFpB(fMKMW_CC#g?F_28^xSO(B81-b>4*b=xAo^-_DkBFI4)!nahb$_{ugn z3k0}m7^Wwi=2w-qR zDJ*sZ2z_IrujyY!c#7P&BL4<3*76-01T;|+wVkk34!poG211-r_es4g?6v>Qkhv}X zR%4DMBUB>>FpH799xQIkZfw*H99)hSFS6vlFs{)3F}hO(?*m@Wrh zyh2FakiYn_tRy{`n5v@cz;P6Tv4v5-svb?@-tXH%=Ti=o!mmla%^8&=a zkGJl+gtHk`VCm&%QxWpzKOrHNz}BOTkz@}PS|WA3`tg1er3_n(=PC$WeAMizx`s+u z?Hy8XD|s{c5*f!IQCD^E$>!y_=TR4OW)PWY0`zWOp^~^3Z({ISljC{0C6ZEEYD6VE zl*&$oY*nhrJSoOI8n8cS_RYtfZ%ElLdGGn(-*n*xpvxzuVC5D96W18=BrLN-Ot;Mc zNxY&&$`HJ6pG_GUQwdIl7!MLe$UNL4kc?Q3B<~&k(vLgERk$^1=puv^OTPXvd4Z%P zt1M~6T>w5MVY@I?16Bj|j#wzw@*ym%X;yos^yyR4F5JKvVr=G`+QkZlW;P z2f<8L@@f%SXqjN+u%{Ili3*UZmWv0ykio?ibR3yvhe;^a{*kMKS6A!n=gG+_%Je%O zDB|A1CP#)I0V$Hh^p2=ml0HbkRLdHF`wck=2Ov;cEpIJS6NltqGTLZ3iXQ@gbTP* ztD{5#@%y{AsGAK`RG35%*0Ob|P|LLpysTC1f~j}>)?yD5CI8`gepWD3XRV&YE~?t@ za7f>{_N3$>M@p7Z(k86a-MZoO$&Ed`d&V7lYWtxgN(-&u(hzJ#tr-C^5n%-7=NCF|W zbF<;WAWuVTN?P6P54i<7hOKTtx$Becxw1p`TDyvUANYEx4S%boS-0BfUc~_K|%sAL9y|hH>TrbxSV^h}mcfBL6CQ3s-LdW0yJZ zov>*#IScz4b`&uf2a5fLal3rFkG$DJ@gG7UI*cnCDG7D3+95&e6*!FB$fQ8w+s^?- z)F4j-c@nb$s5z5Te9!l003Z5d^03kfgh0sXGCeEcNj`nBqGAUP?b?UQ6(nASiP( z9orB$A%7!sEW*I$>HZPE74=;B|C#;r^@}lj9n`|^jCWPA88hsA0)5HUA$u} zNT9f!;Im;nrn;icpU6CJT($I@gLwAap~a3K_KWcI&9r4Z*q`6fkB097r7|B)n!&|% zKbb7oHO2$o-ZGx54}kif)#yK^%qzr~Yu$)tx$eR-TsX1)@AVrl1*fw90?o(uAQU<= zN_%U-pB^R}u=jqB@&nMiqC{S$zV07(UY)LTeIC15C$r3%9K2XG+%Alx`AKa| z94}8pu3RzxZ3=`aYT`L-(h$x zaM%X(TTgl3l>{E(qr0EHNNpR4BN6ns4Dkk=X$ z2`LnqxsAF)_PN;?!V5C@`%YG?yEl7RErdnVRI`?-T;5r+Aqy$6lfY z+glu=OJS2X=G`v*em}!2p&qFRS0sPapNSk4@U(iL`EW#4g@bU_HNrnEJVLE1p7ay#?gO zs_hGhdb8xVvGrN{?kFq^SA}t#U5w;_AElfYxCHFwL@1#PUIq4TTS{or+Oje>e=aJu zIIL?gEJQ=Cp%@3C9s@fhX#s0A##d(FD73l~_|Sir8G#Ae&r}_!A0$ux57SU}1NC;+ zBbSdncvyC%2q#Dv$LXURbLwLvKv2HS;Qr%Z(tl;>l^KMuW?qJ7(~ZWyl%iFzCok6i zJ{&SZ6(TXj*8=nw5Tg&EwSxn@jAl|SERfPQWCsCDC>CGRBL=d{#`D>rSClJw=LZ8s@2ED6x*EX6BI&I>@sRRjPl&)d zxG}0DZhsry;d#rjyzlq59k0r{`4%;c>@_QOns`4m4qi;esGZ3l#UzX1;+Wd44ek_t+!b>ImRLI%3-T zNAzSa>7X-HkkrqX?LEP~wx?sKxeyC2=Ut=d5AMwi7KP1m>Eu{KI% z9M(P+(z%_j0t!3_Pu3N75s9@EVPDjwUnHY|J`k3EgIXo>c{*guyC4~N-l^P8-N*!mNh82q%^pw;QS)p?*-(Z{|GD||)q zXdxTr)y7RJdeVzWF3zUz1dKfJh>eh7itwhUrjcApP%Sh*61kc5sJf8NHUT5b9XFb( z5L_)&>GPQo6(#S#9+rAC>N$Qpseci-(1fw#UY3l+l~28*!p!wzXj5tEyvWW1+Xn-6OB5j?IhZJ3< zIAFiu9$q|H8SX{SS5P0VJkPMkFwms()Jh?dtj%&SGA%#K8_d)va7~z#U^OIgg?JlP zrFwZ*uZ^M;QOYXzfl*Zd`cu=TK`|@8gHpKC)zYSg(gV^7jg^2Jh2P8n4>B6$H4O-5%*$uA0z#OXD?rLz>o%7wRrj1XI&KpY3{z` zGK-%A)q))QiaN(ASKU@{S&5%y@2vx)V=C>tzpi(l8YLHO7DYKv6+DywD;48;1?Z_t zvTQL|9fsnt9NMrUd%LY`B$z^lc^6*XnV`adE4Dw zS*Dn!_fZ+J*0_~}lc!>`WY{n-XJ(+Nm5d$pQ*0diy!$M}%fcM-0lRT|uwX+>02a7z zddZd7!*NWhY?5NV-PCjJ>jj_=s)lLLu8!QXN_m5M54r}&co)hE)obALfj%ukwIs_6 zlGtVD+-I|G&9qAE3?xhQJCqnM>04~a(Ei?T?+1Gs6#)%E;ZI`u4M(=H_o;^5q;NA}=f6G+z7Ha~Pp27Vp09qPt zdD#RtkULbK)W zJAv3UkOD$relk&B7c3eRL%0|;0ZZ$$A9J!Q6r(8K#&q- z>32yU@w@M{x=@zOzu7iNAx%PmNj|ndFNUi^ka(h$I5RnkAE@Ls#si3Z-#GI3ZI2X6 z2IvH!#*hCvs2%r)E+Iml&RU>FBc91v9x3tHUiIuSSodEx!ew+o^iu}Bx$Mm`|Xm{iJX0jW{vrA%$O$n614|w17={BD*I>Vao9X7lI zz4~knv|plK?hKhegApr?8?0L_nDUm`e@tFr&~DEX1Sm2`%;a6Jt&P841H}xhjm85p zK-Ax`17e_~D?(BJnaxj3vyaI4a^ZgxeSH{XT%`4f?|EGMTFyNEr*_eZk`&$Wdb_Co zilGgBG9(w3da&N=xvKxe12%RGox6QIKBx4s2)a9(gCqSNj2b`_N9s{fZGke%4ZYGB_PHBx-@7Zcm*vwa$ncqr}6(RD97}xC+RI?iKK<5qWJ2+Rz&*~hxUTqrKahK^nH#^3FR8(_jRI`XW>@v-+j^ z{&sD}sPi54?drRgWnWQaKfy#+M0i*Q6}5hatI%C9*u5>k!|e18xGQU}~zeW557P zJ3O%mQ?mi>+tgD~R{&d&XI#4)gZArAF$UCBw71KTyTr~Er~IVmY>Iv;+Nz0K=go)c zOuT7~-qmqiF+ke~Sp(inp8^#*Ha(ZTA~P?A1pprHD`zQD0O-8&$$^ga71nBDrdcpi z5`RB!L@nnV?_Kseg&e1V3AWvCnh&8b6R~ej`HyviIE3XAzec(YAMUj(S1Xc1bD z(8|Iv78}M<=2m1`h5eEvz{T+UE4I)pC1iH3U{^m4n3U+3$?I2z;CUZ)5hpy$TS#45 z6!1zlG|~H(Io1=+#IH&D)$W#&02E2(B>$OT2x>TahPIa&Ft zdT?Pv5W8t=xsv4;xxHxMuRC1v%S~Jvup+f*yS9)WhJhE{HH}U5P$kwYoFp|NWOKao z&7FDp^WHU2PAa5uK;jgZ`$d zTiYUp9)4Q=<=d^+n;Z8+POjP?G^u(Zsdal4w%0zf88&O7!YU2>MRp&b8O+lU^2TSWB zTt-=QH4Wl4;~<7SJa;o@DFa&ZKckfx#N?nFQ&b7BHBr6o#f_^+Jg(ZmANmEFS?g`? zuY6RIQcv4n_1hVB1*2-8fq2Uopo)ri)hhXrn1*>(Ps0QyI^u1@)ceDZ?xUdss}1<# zs(wdhRgCB_I&rd6x_v9@Sx&U}|EAP*{f*KfWn(bDYERcF%)Y5bB%WzWTo)q;u7FS5 zI8g5IW>DXgHzJZ5;_US4`V-}973Sf|wx4(}-oCZdOLN~xEd}gsRn&#i3;yKSyT_Hg zQFpmw66yjgba(ytwjtuP8uN@#xfDg+^L zSuDRU1I`F!xRS2hbjf*xM;uxJ-H6B5gS^y_?9Da z&EE)Zr;=lGpax5g%W4ZN5`@JoNOCrTKSonwmFS1iB8}%bv*g(fE1OCyFNhW2-ckFJ zB@MfM8n+{S3}tZ7@gswbEqUPu37q&%+3Dq zP#%Af{i>y8TY2e1jG9xktXdr>^xOH0IP=;*!KYtUm!43K9|yP+q2B+_ZOzgsh*i|- zLKXqG7JDOvHXWadhr)C|W)_zhw1bGXUC`zra3A!epLW-NugqLuL}Y`SCX*yL`~3o! zKww3+ul3AQI=nHNc6@#Sl~tW@UyVHVOvh1c)1WbNh``y zqeuW8wXP7@;l`tIgI)$S{iQW5z8%|eHG6Q5;$;qUlTldD^ZKZQB9H&`)y*N#@xWVx zJSjgCwNELs7!4-D^1~5|+@fYr9%_Kbx8h81?k6tc@QdBD$Jo8O|0kDIiS>UO2d|)b zJ)SL|z@fap=j!5Siv&oHsf-Q(Db5<0H&R$;k>#gHXYjcH?@Ir_OaA}m e!9ivG7m?vSTbMJxP7KH?2#}Fb6t5OF4*q|Gaq2Jt literal 0 HcmV?d00001 From 1da937d6397cd6c7bcf01840fcbad26bf9e39c5b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 14 Mar 2016 16:19:55 +0100 Subject: [PATCH 29/80] Add commented code to generate image output for validation --- test/acceptance/analysis/analysis-layers.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index a0dd0ad9..7665849e 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -97,6 +97,9 @@ describe('analysis-layers', function() { testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { assert.ok(!err, err); + // To generate images use: + // image.save('/tmp/' + useCase.desc.replace(/\s/g, '-') + '.png'); + var fixturePath = './test/fixtures/analysis/basic-source-id-mapnik-layer.png'; assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) { assert.ok(!err, err); From 6db48a24b8c145d3535769658b8672a1f7ebee07 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 14 Mar 2016 16:42:51 +0100 Subject: [PATCH 30/80] Adds test for analysis with no api key --- test/acceptance/analysis/analysis-layers.js | 22 ++++++++++ test/support/test-client.js | 47 +++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index 7665849e..d78d0077 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -109,4 +109,26 @@ describe('analysis-layers', function() { }); }); }); + + it('should fail for non-authenticated requests', function(done) { + var useCase = useCases[0]; + + // No API key here + var testClient = new TestClient(useCase.mapConfig); + + var PERMISSION_DENIED_RESPONSE = { + status: 403, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + testClient.getLayergroup(PERMISSION_DENIED_RESPONSE, function(err, layergroupResult) { + assert.ok(!err, err); + + assert.deepEqual(layergroupResult.errors, ["permission denied for relation cdb_tablemetadata"]); + + done(); + }); + }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 02128e36..2f46c699 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -225,6 +225,53 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { ); }; +TestClient.prototype.getLayergroup = function(expectedResponse, callback) { + var self = this; + + if (!callback) { + callback = expectedResponse; + expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + } + + var url = '/api/v1/map'; + + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + expectedResponse, + function(res, err) { + if (err) { + return callback(err); + } + + var parsedBody = JSON.parse(res.body); + + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(null, parsedBody); + } + ); +}; + TestClient.prototype.drain = function(callback) { helper.deleteRedisKeys(this.keysToDelete, callback); }; From a444b80c96e89fed886c48eb3ac4a0faaff8fe96 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 17 Mar 2016 12:44:51 +0100 Subject: [PATCH 31/80] Fix typo --- test/support/test-client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 2f46c699..23a7741e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -263,7 +263,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { var parsedBody = JSON.parse(res.body); if (parsedBody.layergroupid) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupId).token] = 0; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; } From ed84ed8475137b64fdeabb23f28f62e2e182d64f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 17 Mar 2016 12:45:05 +0100 Subject: [PATCH 32/80] Test client detect png request based on regex --- test/support/test-client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 23a7741e..b79ce9e7 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -198,7 +198,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }; - var isPng = format === 'png' || format === 'torque.png'; + var isPng = format.match(/png$/); if (isPng) { request.encoding = 'binary'; From 5bd30b6b5fd1d1369939ecfd6a300452ccd51740 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 17 Mar 2016 12:50:42 +0100 Subject: [PATCH 33/80] Analysis layers adapter skips analysis if there is only source nodes --- .../models/analysis_mapconfig_adapter.js | 36 ++++++++++ test/acceptance/analysis/analysis-layers.js | 67 +++++++++++++++--- test/fixtures/analysis/buffer-over-source.png | Bin 0 -> 9439 bytes 3 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 test/fixtures/analysis/buffer-over-source.png diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 99d34f5a..6c437849 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -1,4 +1,5 @@ var queue = require('queue-async'); +var debug = require('debug')('windshaft:analysis'); var camshaft = require('camshaft'); var dot = require('dot'); @@ -29,6 +30,32 @@ function layerQuery(query, columnNames) { return layerQueryTemplate({ _query: query, _columns: skipColumns(columnNames).join(', ') }); } +function replaceSourceRootQueries(requestMapConfig) { + var analysisToRemove = {}; + var analysisSourceRootsIds = requestMapConfig.analyses.reduce(function(sourceRootsIds, analysis, analysisIndex) { + if (analysis.type === 'source' && !!analysis.id) { + sourceRootsIds[analysis.id] = analysis; + analysisToRemove[analysisIndex] = true; + } + return sourceRootsIds; + }, {}); + + requestMapConfig.analyses = requestMapConfig.analyses.filter(function(analysis, index) { + return !analysisToRemove.hasOwnProperty(index); + }); + + requestMapConfig.layers = requestMapConfig.layers.map(function(layer) { + var sourceId = layer.options && layer.options.source && layer.options.source.id; + if (sourceId && analysisSourceRootsIds.hasOwnProperty(sourceId)) { + delete layer.options.source; + layer.options.sql = analysisSourceRootsIds[sourceId].params.query; + } + return layer; + }); + + return requestMapConfig; +} + function shouldAdaptLayers(requestMapConfig) { return Array.isArray(requestMapConfig.layers) && Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; @@ -36,7 +63,14 @@ function shouldAdaptLayers(requestMapConfig) { AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, callback) { + debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4)); + + if (Array.isArray(requestMapConfig.analyses)) { + requestMapConfig = replaceSourceRootQueries(requestMapConfig); + } + if (!shouldAdaptLayers(requestMapConfig)) { + debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); return callback(null, requestMapConfig); } @@ -77,6 +111,8 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r return layer; }); + debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); + return callback(null, requestMapConfig); }); }; diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index d78d0077..2d089cfc 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -53,11 +53,12 @@ describe('analysis-layers', function() { var DEFAULT_MULTITYPE_STYLE = cartocss(); - var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 }; + var TILE_ANALYSIS_TABLES = { z: 0, x: 0, y: 0 }; var useCases = [ { desc: 'basic source-id mapnik layer', + fixture: 'basic-source-id-mapnik-layer.png', mapConfig: mapConfig( [ { @@ -67,8 +68,7 @@ describe('analysis-layers', function() { "id": "2570e105-7b37-40d2-bdf4-1af889598745" }, "cartocss": DEFAULT_MULTITYPE_STYLE, - "cartocss_version": "2.1.1", - "interactivity": [] + "cartocss_version": "2.3.0" } } ], @@ -82,8 +82,44 @@ describe('analysis-layers', function() { } } ] - ), - tile: { z: 0, x: 0, y: 0 } + ) + }, + + { + desc: 'buffer over source', + fixture: 'buffer-over-source.png', + tile: { z: 7, x: 61, y: 47 }, + mapConfig: mapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + } + ], + {}, + [ + { + "id": "HEAD", + "type": "buffer", + "params": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + }, + "radio": 50000 + } + } + ] + ) } ]; @@ -100,7 +136,7 @@ describe('analysis-layers', function() { // To generate images use: // image.save('/tmp/' + useCase.desc.replace(/\s/g, '-') + '.png'); - var fixturePath = './test/fixtures/analysis/basic-source-id-mapnik-layer.png'; + var fixturePath = './test/fixtures/analysis/' + useCase.fixture; assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) { assert.ok(!err, err); @@ -110,12 +146,27 @@ describe('analysis-layers', function() { }); }); - it('should fail for non-authenticated requests', function(done) { + it('should NOT fail for non-authenticated requests when it is just source', function(done) { var useCase = useCases[0]; // No API key here var testClient = new TestClient(useCase.mapConfig); + testClient.getLayergroup(function(err, layergroupResult) { + assert.ok(!err, err); + + assert.equal(layergroupResult.metadata.layers.length, 1); + + testClient.drain(done); + }); + }); + + it('should fail for non-authenticated requests that has a node other than "source"', function(done) { + var useCase = useCases[1]; + + // No API key here + var testClient = new TestClient(useCase.mapConfig); + var PERMISSION_DENIED_RESPONSE = { status: 403, headers: { @@ -128,7 +179,7 @@ describe('analysis-layers', function() { assert.deepEqual(layergroupResult.errors, ["permission denied for relation cdb_tablemetadata"]); - done(); + testClient.drain(done); }); }); }); diff --git a/test/fixtures/analysis/buffer-over-source.png b/test/fixtures/analysis/buffer-over-source.png new file mode 100644 index 0000000000000000000000000000000000000000..06aeb50aea7fd2de65c6ceffa0296d37905f6216 GIT binary patch literal 9439 zcmXAvbzD?U7sv0?&C(z#9nv7pE-Bq1Al)6(xum3kfPi!_pdgKOC?zPd0@6sAG)TY8 z^ZvE_**P=!&g{%NzwkOFvFz_+9jr>!Fa%fyP%#AL z7}08E86*=7@?AM}11+C%tEtjALM57J>kL*HXP#_JS4i0((oHXR>yM-d&gBm--IxR& z9A@4MlzgkdFkZf%HeP-`o@ahval|y%FNWX87ei>pHyH)~Lt!U#Pv(?ezQV8zD(xF8 zPg+iR|Go|p(Lav)XLIaD$<3vZ&iChBFc4*E(3NK1L&phgQxD%+q=Vasi2i8f0VB|v zjP-zO>W{g(F+@b)IE7!W*`4W4)kpeD+dMKTZ?n`nJoeWsNlk7BygSQZre-8&^(zET zqYWtAn@3L>+kz@SS5;r=;$kvU_NY${Wf4Ek3^4#6IEo8gNnE?{Gx(C}D+|f-6g|N0 zdN-LDzpCA|zlV8{^?dGStPe=zV@*nXSg5;57Fn$?dbDEr@T4Ng+(zUNxk?v(L@6*w z7o=o%leG2X?q*r)v^7>3QCmcgqxcynH^|A&A#CRVKG+;h78F&kkur1dZ#;9mJIdh{ zZh=OUm+92pkFVYdX*2*1xRx4gujwzYPPg+-lbRLp37ssFCLScR7{`)jOI&jtu%Q?q z#@pS#&<8b#UKicQ<{yRRXo?=;VDIYE)(+P*lJy`(T=ek^^L}~nEnmw-da0v=7U5cb zqq<~cEXrOCw;~T!zu9m84Q8XN7kpyRANG_4+G++>6+XC9tMNkV&EhvqyUpJ zgPshOhw@n@c(5(`I^^)O67WjSD>L)QC}G>T?`6^q(h`<_?Mq>ORv`g z9<|jYow(}SZ@FKEg4Nyi65}kYT;2QZ(uz@xIcrs9RzTv9o7~;ur^$wZiAc=dKCbso zO;Q4V?dk<$nr?xT-Es|P?;n;t(P3kBSN@+QkL6z5wVm)6*{aI^`Mao%z#XwDMLl+% z`nw(8NxPUsOfBTv9^N^Tbo?{!$T|Dc)paqkg?`EqxPbM4fA8%`p7Za)jT$0LrHhV< z%72SjT^?A#Z*SMSHSnzxulD6n^&3IAmuw=wLDLc^j!9vEuU`94&t>vvV0tn(zey*B z@n7ul97L$Q7hJ?H`n~Rd*gp&lO>dnaPf_TM#Z#5ba)=GBd46ssKsgNLbT{k=dsh8*7ZKgI zH^jSq%!x zBnm7;a2j!jab7Gy83#yVGMb2p=pItLl0w;EyXk$kTA++7k}RF5g?~2qX5Z*e2H*%Y z!XpAbsS?9s9Y986s4+Wi=&fc>CB^$*O02wdpTjrgqXgQA6bGcR01I>O^FIQVcQUWK z?;Gjo60dYYG!m~=I(zUG%~KBV`_eMumvPNFFfJ5+Jq(rUfcNVpO6VR%k~x8?REhr3 z_f=Iq5N=X{m_`&YtB6y)y4ttu74Y>Erzh*87(8PvQ2DsU@N@4c4(9I_I$t0US*zLk zM~tY4g1z<}VEFz-gF9Q!ytWPGs?O22vDGwCE*& z;Y4u|yWm?T88lo4>G^5A%%>d~d^CN&u&qcvVlB##%#;KmOl|L(x9;%EH`6#kww{uv z#-88op)2qzvAQPEmx~Gv zbNB>+U#6;16g7%^!5zZfH#F@+-PAwU9lJWoSi>=M?>8whYqbsu56FNj(jxMwzn0xc z1I4V!3OGn+j9GPXY2|2pSny|r&hN*~K>nV0no+Gpg>WWgs5UNrB%Q(UE8a?5oiKF1 z;MiwsD$;M=ipPGNi@cH>q`06%qu%}}$CKQ~1)M$cXlqX)iRuMUFTY&efOuBduYg9( zo6!xSI}6VpMi+(<$mIED09=?*k0}ffrlR$$3Lbi!RsGiB&c(Z2wZAkM$uIzDt07>= zCM`zBe6&1$vH%x~aSALjZg-2$7x9N-9aD5ZFZjNmy!2brxD;EYqs;9@3q*-AhYZ|b zme=C|ZV-___fi{2Px~Z8*PTgYOUYlxH1Ktczm9*^MN{|=r|_WiMJM5nPpXpYUl+KE zJa;4B7I2pwicJW;GfttZ?Ct7R zKUVeLh3X+Ajo_B!&ELa>qUC$mDB~zZ->4EDJEwDpXw0jaT!t?wd;^dQgR0s$`!BmM zVvs>rmU}#L9r$)ILu}EhTm@(q8J?>_S{&(qIIcLzgn@(sqDmER35^sDP|u&_vD#tz z^3hefD1GC;#kV0*v)`(u*1gui!=zA#XZP5|O$473ewo10b5bd9%`iT)DU$-bH+xnJ zKJnX}{hFY#9&tmjSYxx?;uf5qt_!AqLW8hfLc=7bU|D(;r&T*VJZ~W8neO*Qk~1|> zVGU&IHodG#8*$Q$z? z!$|Tq>fO+GGxzcdH`G<0a3fv&dmM@PD-dOa;QHnp;=+5;ZsON(YUB{IKenmuEMo8hzh1+v&;2cchGKMRfh-6STWaP4hNVC77VI&2)pDyNuG#nMfaYb zb}J|MRtz4h>IvmFzWFXh3e*=Bn?OT3E(C1k2yWDQMLI*~*3eWgD6T(cSL_NQWAA^4>w*?{bJPfdcLbQLHkpP&K~p7E z_HAWvQ#QZXK(}VQ0)KI*7nPIyDh6jMM0bT^;0drMz3#vzTt>-dalr>y{8rBS&TEX;Aaj+A}{)})I zWu>m@eTL1pvc5$qMyX~4!=X?UOb`RZMR)-1VZmQY8)yanJ8&v}q8<6xxRnYO_4tY1 zYMhf(fO2|1I}kp?9$#3x;|ARt@v1a>Uz9#R*b04x#EZGG`zcZtSHFae>!LF_6G#h0+?L!a zRo7e5H-QBw<@QH1GFDqMALO}JrFJTj2ed0)0;2*dWWDNbA@!)bDXf2t@-?aXOzEP7 zKM3kZZFb7uPlQ@K%*j6uy|pB0IughY*VpYNR4{{Y;GF4!R!Tf^3-2Ae`8%wPa{h~m z3rnyZT$b+#bxF|?ZAkePO+tY3_RMycJLM~>RN(e2A+~nn7`29fqn?e@ExSCmGTDMx z%b4Sxj{4~oRpYM^$oJNb{35G0DKDSRnIj+7uUX_=7a-70Y5RF@+2l$WDR@X*NY}2k zQGJu!cyWJ9@&qY0){a~g@e_EwHfHlmC}KFe@i~lUutsxXu}~|ZEKBFsBm;)>Ve3?j z4b`byT4CXVQ3gA|JGOgKeOi-R^M1!IC&#c)kbJRi<3e4?I{Ktn{|%=H^Q+@tnO6Mt zbX;rsi_^DvO!c03!YcjT9g4FI7Y8en&lBE`06em%3Hyriu`&ige8WNA8#J_K5*e2( z-x{CtZQ1Q4!+y?~S}Ptczn`2#kcu3E9`)~!naF_j!A@>HidHVzzXjd&mHRMsG;q5q z`swbWdOBa>)6WA)-vzB;=3oW~P;vF(Kt2D^?!W&W=gdvwxf_+)8TjT}LYHM99jieh zH;OZ6(tF~!)|N4fT9q*N*soGgRu#{pC&^Q%3LwwZ z&63uPCUrXZHZk`lPFSkkZ~QE@RmRto56PvZdR5xoG{2PpJdbpjmm#RwOu}iMC_Ko@ z4%&Zb65lfqEyT7K1+!y+7);g^Wl;k&zdWn#9wwPHv=4T+_xYt^nvafEFL3(WIrlB~ zk=|qc^VBbtcU*pN0ah{eSm)3&I<|FKb&0&qkz~b--dxSaK=y#?p`iy*(&t zCT#aeN8iy#EpA1uG{LlxEilF<$DF1kf+k7+T@rE;mHsv9V!M4Quz0uGq-9@h8)C2U z+fbY{JRf$6RY$bBqpL8)g3bM5IlQlxH zaQ%f>S8Y{oY^hJ84YSzE$ef)5nSKlFm9<9E$EBFqtbM6#2?qUS{SbtLrxX>++8$be zGg|6d6O*v3@fBw~T`Qfmg+SE7=F(Yd0J2L8h+%d98ta%iR*d6$iw5due6 z5!zTn9Tv~;WUEzfq?|GeeyZ9nu57)4;a<%OKkY;KB$7~q&D3fn@5C6&G)F#!DZecJ z_Q^zZ#9JGEpoYW~w}EW$aACoHq&FdM<1%bnK7*S16LXtc)#Zm%iB&?sV8?1Is72xGuV zRn6StW|uC8nT<^`OIBtnzifzM+bdBMbtg1R0bHsR;{Ci9jPX{3n99*a#}R zQE=gSF9^usUvs>RHC4aAB*kQ*4}<=`Snm(S#9C!K*6=%NjF~4C0d~LX#%;zewCQOO zuVZQ@OKo@9D!^21tCW;r0$ zb}T|~2$}ly!m#*lS#t}lmEP6eN>2m3Q^2&b)?e5ABrl zuFu^Q>Sf<{E<8ccKG>dqnt^8O#rkXxT??8^FxP`a$6D z?#D-8nz(>CAYa*L{;i4WPr5$VQ_k;5b5$Ime0I`+qiH7FY$oo^SaRR&enWA7i8MdH zg{SWdbK=jijv8Pr{)}P}LhbdTE&Gxy`qXM{Of|`o_gxst=~m(j{C#8 zWBl_L{xuQ|_l5CXV|Ooq>F~6HVNno-8h14((`Un^D{t>CaK-OQgW2W)9nKV|#6#KC zo(4vhGTvALOR+>t!Z-qZ9rpu!9e>^N;xp=RGhydQ30~>td?$r@Zo>%2a+BZnJbTe- z^;|Q*HA`je-lj+F;hy1|XU!4;TE4FNt4^8jW@QN#5k9~`1Xj2&KsMV{>d(M zN~qj=SQBVCRkwL18wGEQYN)%=th+>R_TDu)gt(BCldf*T|Hd;DbgU^j@8MlZkm9fR z{Hxr`%ck_iawH;GlS;iK$}_eS{&ioljs}!PnGl@k3X<51|IQzQ1rlQxt5vHDHvAy$ z8AJO?vU_F6s=fa8w?DGq#0PdQp!^|#Z)IiEd|I%!ul#?jfJ=_hk47E? z_o9435sN&Yk&1kCJFi5#5ss4gIgK(>al<;zzb41ov1VA#7X;Eu{u71^rTDwbYSH3P z&`TbX+v$kf+t~TS|JzrdEK^$00KL)P%kFtC@wjMX=L#iu) z1I`ZFnO-#n!8t#w)*%X07Ga!PX(m03G1to_lXzFb0wk<2P&j4vj*r_-v=%zLQeJ~@ z6(r~XzY@jP37$1yXXajhwX*&~R~R;XF@#lvSxBrc0c~tdJh-p9L6vGc}=&X)SJ<-v-f{ZT0Ehm+?e^1fsC}K=^LsW13ssR&?yZgd4z#b ztC46W-Y74DSy8^6oz>Ugl^b>_3~6YdrA{q%)1F8%IMjirNBaI}j8LR(D_2SwT!9e` zl}gnh!B^tAHA|4e-g^1&TV zV{>|U@Ra=kXeLHJZ4;mo)9ZP;Kb?O@x}LVfx0)~0Ewk$hD~f(CXW*#psCoDm`Wfd$ zj7!TiX4xdbgmQq%?XT)js8qIF$}ZJ7#e^F1M10&Vmbi0O3)_C3#k+CI(9*ty&K+zx z+Z6tYVo8;2@v2QRrSgXgUH!}IoW`AbK*D~#pkHKKnpGz9dWF2f zyZvvlPUYQ9f4V5{5G>U1NSxI1(=#M~Nk3RK1)Z0+Ddg7BR@V;=rm{0hNV|3go$ubQ z6IY+~fj026K@~oXmy#Q1KIn+F-6wDG$y4$7wvj|hV*38r&OT)E<|RcJI{aJnARX}# zd)ZR$+sHtx#nRoF9)CREv5u)l-?^%qh5kE~Ulxy)FW4LqqJy(dt6yt_V6JI{IhT@X zOerq%Qu&1>0d&Us_rUpjn{qcN02am`zXbTVu-wb(G*cWmx27FZuc-+UX?v`9AAz26 z_b`%pVZ;N~IVOCNSTBbBr^q|?-4il6+pPMvDF{}7^%b&rVi8qLr@9gwKUKz3h6qCF zr)btEoaImy@*>r6FzsD%81A`fgrdK#_qNysFP!c7FZ5R#ERSUI#ItlPRW4fJ_nzE} zdT5Xs-1j$vu^w?Vg>?LlIe5G#f07+~&>T9BKI)+$*069UE>u-p%-tUw<-^CvZ;jU1 zJmOc$zRwj8CO7IN%lQ=-r;!Ukkbur@+J}(@0mZhra&VQIY5t+TX{}$_4?bAL{f2Ny zV|;SzV1{~~g7qol)FUhr}AO+hH5tKrTNLYD-R9m#pkW%_L z--TW$PxskI>pr`^@N(nLXRrx4^ko>o<>0NurS~6=?!q2fC{huNJ-pq_&*63wq+?9VX65p-R3` z=TvohyxEPpi|(25$TeLxrPYw;vBZCn+(dZMnbU147{ST;aih4HkT}T@)%oBM7m@I1 z&g!O+7nZhfT1P(Nl4rY>;FB1o_TK-{B$|yEoo9%RNQ?I^SEFXv(4h0q;ObK*>KXGr z+P-*ao{tLyi5#?uwUT1~wX#@!ZL=Z6dueDV)a`S~x170B5ZkkDl7ZNlx>MB0)0A{5 zMplv*Mg0!zL=N;}$OU8;p={g>6PeKcTQu}h6wQse8mqk4O@yz(`qTR;J6P13?FmBj-AncJ-|i zD+76?R9#pRbfxz(;gpR@kEm=UMngBF zSHV^yu%wHDRKe711*i8W1Jc~ir!SJ}YCiL%+b7VwjZQ2U@Ro@F%peW$p`KC%G5q1i z`W+TGz7u$HnbB)#uc2(4vf19C`Id$=;k7(coAsA*a8G|II9r5S^^Yl%`yU^Hk=^85 zLK!2-wcUlGuadK~B+u_C9m(hx38@CerEeqo^63?%3CMmwY6MHi3iA6x zIU(Vs@F2~N6(Ms@p!H64yTYAu3U?>NkQO0)Z0$A@Y`pqArB1j(# zR3b83;XQOWG1sU~G|aML96-m7-yvRb#{)FVlGO7QLH#rq$;kele`~30j;e2 zv0#`Aj`T6WFZzy1*~RFLC6^1$9&EO$kt8?_iOWtr|AogkY-yReKSfq^b;!;1E$-5a zcuArDzu!R{ue{EPfvU_ zqgA*7d5yYnkN_Q9R1BgaZ=uamJl)!WDm?=b!_XV<`%0qoJGh7<7zU;Hwkb9{^oEzB zKw}D?{TWygOgiJaZcR61q6 zF|YBLYFiGit4e?v!qmf%AAePoi&(OW2vB_~V$THaG@L8UoY~+}D9}66j!S*D%rk0j zy?273#pvuHL?%zSKeHhK-}t%Fk!O7-&(jW=qE#`qsMFQzX(xLtRRQ7#VLjg@X_=Qw zT>x+7b%Z5_T_4|f%s7DX?RrpwTo5gbiq~rbSP)cBpUFDjz%)ZjA1D$NRPf%q=xI6h zO4vMRB17cJ+h;|yRaMT+ItCU49+PWd3A?+pZX=smZXbky%~y)V)j-WrAun#v6%5IM z>tV~a2J_{R5djO(e$^V;cu!-GpQy{6k>cHqOM!!NPdo zm#4eJzL7uJfuOWE6l0lxCBYkQUo;EeG;TdfVyP5~uNgl-!_+kZ;G>ZwW^UK`dp2t? zc9XF=ZP5#nI;~`hT^!|C#;R$NnRQE63JNTY>M$U?)Y{!wFv#67pvIn#?Exqcxr3Z+ z*zP73b6NRhK9s(uYLnp`UmbbNOk=b1twe3ysH<@=z5B9!OEN?_){OqJuW9ht?ZlJhx@E%YON)d*_lJJdBfVaXe3`p zwiPxro{P7SHIl)Q`Mv#F08+|)`CITEI$XP5L%W<(+w`3V4I=&Vb_eb6(DTJ$+@nMX z@nHqXLbQWx9_2_bb(xuS3x_*!wDldUi&WEVE!w_4Jr;$&RHDux_w-Quv0RL?sui?? z4XddaaOOq7q>B&o5GuSu7CMAH=6NqI$=CDB&765DXE4AYAgY@OZv@I&&s1&j!)|^u z$)@aGe;K^EG`NrIIz{9#Va3u(dA|+p?6l}sgtq?kVCo_3u7#l5dUnS^S`K}ah7^*2 z={>{+TBD&qx~g5_%LvpU(EeEBUgxs~F|Tb@NP^*1?R-A;&QhF617zNL^=rw2A zd}3% Date: Fri, 18 Mar 2016 13:04:00 +0100 Subject: [PATCH 34/80] Add camshaft-reference --- npm-shrinkwrap.json | 17 +++++++++++------ package.json | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 6c7a67a7..912f2100 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -90,7 +90,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.2 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -532,6 +532,11 @@ } } }, + "camshaft-reference": { + "version": "0.2.0", + "from": "camshaft-reference@0.2.0", + "resolved": "https://registry.npmjs.org/camshaft-reference/-/camshaft-reference-0.2.0.tgz" + }, "cartodb-psql": { "version": "0.4.0", "from": "cartodb-psql@>=0.4.0 <0.5.0", @@ -631,7 +636,7 @@ "dependencies": { "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.6 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -695,7 +700,7 @@ "dependencies": { "unpipe": { "version": "1.0.0", - "from": "unpipe@1.0.0", + "from": "unpipe@>=1.0.0 <1.1.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" } } @@ -810,7 +815,7 @@ }, "type-is": { "version": "1.6.12", - "from": "type-is@>=1.6.10 <1.7.0", + "from": "type-is@>=1.6.6 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { @@ -820,7 +825,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.6 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -1035,7 +1040,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.2 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { diff --git a/package.json b/package.json index 81c23080..16afaa12 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "cartodb-redis": "~0.13.0", "cartodb-psql": "~0.4.0", "camshaft": "https://github.com/CartoDB/camshaft/tarball/master", + "camshaft-reference": "0.2.0", "fastly-purge": "~1.0.1", "redis-mpool": "~0.4.0", "lru-cache": "2.6.5", From a769545e39e2e700ed0032bfc339caae96487557 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 17:16:51 +0100 Subject: [PATCH 35/80] Rely on camshaft.reference for now --- npm-shrinkwrap.json | 9 ++------- package.json | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 912f2100..efd63cee 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -532,11 +532,6 @@ } } }, - "camshaft-reference": { - "version": "0.2.0", - "from": "camshaft-reference@0.2.0", - "resolved": "https://registry.npmjs.org/camshaft-reference/-/camshaft-reference-0.2.0.tgz" - }, "cartodb-psql": { "version": "0.4.0", "from": "cartodb-psql@>=0.4.0 <0.5.0", @@ -636,7 +631,7 @@ "dependencies": { "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -825,7 +820,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { diff --git a/package.json b/package.json index 16afaa12..81c23080 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "cartodb-redis": "~0.13.0", "cartodb-psql": "~0.4.0", "camshaft": "https://github.com/CartoDB/camshaft/tarball/master", - "camshaft-reference": "0.2.0", "fastly-purge": "~1.0.1", "redis-mpool": "~0.4.0", "lru-cache": "2.6.5", From 697749b204420eb69261d8d0ed1c62da0254d3b2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 17:21:43 +0100 Subject: [PATCH 36/80] Add timer helper --- lib/cartodb/stats/timer.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 lib/cartodb/stats/timer.js diff --git a/lib/cartodb/stats/timer.js b/lib/cartodb/stats/timer.js new file mode 100644 index 00000000..0ef725ab --- /dev/null +++ b/lib/cartodb/stats/timer.js @@ -0,0 +1,32 @@ +function Timer() { + this.times = {}; +} + +module.exports = Timer; + +Timer.prototype.start = function(label) { + this.timeIt(label, 'start'); +}; + +Timer.prototype.end = function(label) { + this.timeIt(label, 'end'); +}; + +Timer.prototype.timeIt = function(label, what) { + this.times[label] = this.times[label] || {}; + this.times[label][what] = Date.now(); +}; + +Timer.prototype.getTimes = function() { + var self = this; + var times = {}; + + Object.keys(this.times).forEach(function(label) { + var stat = self.times[label]; + if (stat.start && stat.end) { + times[label] = Math.max(0, stat.end - stat.start); + } + }); + + return times; +}; From b3bbb9d97a4efd751fc406381c09afcc5cf2b87a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 17:22:02 +0100 Subject: [PATCH 37/80] Initial checkin for dataviews It only supports histograms. --- lib/cartodb/backends/dataview.js | 167 +++++++++ lib/cartodb/controllers/layergroup.js | 36 ++ lib/cartodb/controllers/map.js | 11 +- .../models/analysis_mapconfig_adapter.js | 54 ++- lib/cartodb/models/dataview/histogram.js | 318 ++++++++++++++++++ lib/cartodb/models/filter/bbox.js | 120 +++++++ .../analysis/analysis-layers-dataviews.js | 124 +++++++ test/support/test-client.js | 107 ++++++ 8 files changed, 928 insertions(+), 9 deletions(-) create mode 100644 lib/cartodb/backends/dataview.js create mode 100644 lib/cartodb/models/dataview/histogram.js create mode 100644 lib/cartodb/models/filter/bbox.js create mode 100644 test/acceptance/analysis/analysis-layers-dataviews.js diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js new file mode 100644 index 00000000..0e52aea3 --- /dev/null +++ b/lib/cartodb/backends/dataview.js @@ -0,0 +1,167 @@ +var assert = require('assert'); + +var _ = require('underscore'); +var PSQL = require('cartodb-psql'); +var camshaft = require('camshaft'); +var step = require('step'); + +var Timer = require('../stats/timer'); + +var BBoxFilter = require('../models/filter/bbox'); +var Histogram = require('../models/dataview/histogram'); + +function DataviewBackend() { +} + +module.exports = DataviewBackend; + + +DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { + var timer = new Timer(); + + var mapConfig; + var dataviewDefinition; + step( + function getMapConfig() { + mapConfigProvider.getMapConfig(this); + }, + function getWidget(err, _mapConfig) { + assert.ifError(err); + + mapConfig = _mapConfig; + + var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), params.dataviewName); + if (!_dataviewDefinition) { + throw new Error("Dataview '" + params.dataviewName + "' does not exists"); + } + + dataviewDefinition = _dataviewDefinition; + + return dataviewDefinition; + }, + function loadAnalysis(err) { + assert.ifError(err); + + var analysisConfiguration = { + db: { + host: params.dbhost, + port: params.dbport, + dbname: params.dbname, + user: params.dbuser, + pass: params.dbpassword + }, + batch: { + // TODO load this from configuration + endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', + username: user, + apiKey: params.api_key + } + }; + + var sourceId = dataviewDefinition.source.id; + var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId); + + var next = this; + + camshaft.create(analysisConfiguration, analysisDefinition, function(err, analysis) { + if (err) { + return next(err); + } + + var sourceId2Node = {}; + var rootNode = analysis.getRoot(); + if (rootNode.params && rootNode.params.id) { + sourceId2Node[rootNode.params.id] = rootNode; + } + + analysis.getSortedNodes().forEach(function(node) { + if (node.params && node.params.id) { + sourceId2Node[node.params.id] = node; + } + }); + + var node = sourceId2Node[sourceId]; + + if (!node) { + return next(new Error('Analysis node not found for dataview')); + } + + return next(null, node); + }); + }, + function runDataviewQuery(err, node) { + assert.ifError(err); + + var pg = new PSQL(dbParamsFromReqParams(params)); + + var ownFilter = +params.own_filter; + ownFilter = !!ownFilter; + + var query; + if (ownFilter) { + query = node.getQuery(); + } else { + var applyFilters = {}; + applyFilters[params.dataviewName] = false; + query = node.getQuery(applyFilters); + } + + if (params.bbox) { + var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + + + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), + function castNumbers(overrides, val, k) { + overrides[k] = Number.isFinite(+val) ? +val : val; + return overrides; + }, + {ownFilter: ownFilter} + ); + + var histogramDataview = new Histogram(query, dataviewDefinition.options); + histogramDataview.getResult(pg, overrideParams, this); + }, + function returnCallback(err, result) { + return callback(err, result, timer.getTimes()); + } + ); +}; + +function getAnalysisDefinition(mapConfigAnalyses, sourceId) { + mapConfigAnalyses = mapConfigAnalyses || []; + for (var i = 0; i < mapConfigAnalyses.length; i++) { + var analysisGraph = new camshaft.reference.AnalysisGraph(mapConfigAnalyses[i]); + var nodes = analysisGraph.getNodesWithId(); + if (nodes.hasOwnProperty(sourceId)) { + return mapConfigAnalyses[i]; + } + } + throw new Error('There is no associated analysis for the dataview source id'); +} + +function getDataviewDefinition(mapConfig, dataviewName) { + var dataviews = mapConfig.dataviews || {}; + return dataviews[dataviewName]; +} + +function dbParamsFromReqParams(params) { + var dbParams = {}; + if ( params.dbuser ) { + dbParams.user = params.dbuser; + } + if ( params.dbpassword ) { + dbParams.pass = params.dbpassword; + } + if ( params.dbhost ) { + dbParams.host = params.dbhost; + } + if ( params.dbport ) { + dbParams.port = params.dbport; + } + if ( params.dbname ) { + dbParams.dbname = params.dbname; + } + return dbParams; +} \ No newline at end of file diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b6b4d640..fbf8da92 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -7,6 +7,8 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); +var DataviewBackend = require('../backends/dataview'); + var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); var QueryTables = require('cartodb-query-tables'); @@ -37,6 +39,8 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; + + this.dataviewBackend = new DataviewBackend(); } util.inherits(LayergroupController, BaseController); @@ -78,6 +82,38 @@ LayergroupController.prototype.register = function(app) { app.get(app.base_url_mapconfig + '/:token/:layer/widget/:widgetName/search', cors(), userMiddleware, this.widgetSearch.bind(this)); + + app.get(app.base_url_mapconfig + + '/:token/dataview/:dataviewName', cors(), userMiddleware, + this.dataview.bind(this)); +}; + +LayergroupController.prototype.dataview = function(req, res) { + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function retrieveList(err) { + assert.ifError(err); + + var mapConfigProvider = new MapStoreMapConfigProvider( + self.mapStore, req.context.user, self.userLimitsApi, req.params + ); + self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this); + }, + function finish(err, tile, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET WIDGET'); + } else { + self.sendResponse(req, res, tile, 200); + } + } + ); + }; LayergroupController.prototype.widget = function(req, res) { diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 749a10a9..0b793735 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -155,7 +155,16 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { apiKey: req.params.api_key } }; - self.analysisMapConfigAdapter.getLayers(analysisConfiguration, requestMapConfig, this); + + var filters = {}; + if (req.params.filters) { + try { + filters = JSON.parse(req.params.filters); + } catch (e) { + // ignore + } + } + self.analysisMapConfigAdapter.getLayers(analysisConfiguration, requestMapConfig, filters, this); }, function beforeLayergroupCreate(err, requestMapConfig) { assert.ifError(err); diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 6c437849..dbc55284 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -31,19 +31,13 @@ function layerQuery(query, columnNames) { } function replaceSourceRootQueries(requestMapConfig) { - var analysisToRemove = {}; - var analysisSourceRootsIds = requestMapConfig.analyses.reduce(function(sourceRootsIds, analysis, analysisIndex) { + var analysisSourceRootsIds = requestMapConfig.analyses.reduce(function(sourceRootsIds, analysis) { if (analysis.type === 'source' && !!analysis.id) { sourceRootsIds[analysis.id] = analysis; - analysisToRemove[analysisIndex] = true; } return sourceRootsIds; }, {}); - requestMapConfig.analyses = requestMapConfig.analyses.filter(function(analysis, index) { - return !analysisToRemove.hasOwnProperty(index); - }); - requestMapConfig.layers = requestMapConfig.layers.map(function(layer) { var sourceId = layer.options && layer.options.source && layer.options.source.id; if (sourceId && analysisSourceRootsIds.hasOwnProperty(sourceId)) { @@ -56,17 +50,61 @@ function replaceSourceRootQueries(requestMapConfig) { return requestMapConfig; } +function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { + var analyses = requestMapConfig.analyses || []; + + debug(JSON.stringify(requestMapConfig, null, 4)); + + requestMapConfig.analyses = analyses.map(function(analysisDefinition) { + var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition); + var definition = analysisDefinition; + Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) { + definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] }); + }); + + return definition; + }); + + debug(JSON.stringify(requestMapConfig, null, 4)); + + return requestMapConfig; +} + function shouldAdaptLayers(requestMapConfig) { return Array.isArray(requestMapConfig.layers) && Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; } -AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, callback) { +AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, filters, callback) { + filters = filters || {}; + var dataviewsFilters = filters.dataviews || {}; + debug(dataviewsFilters); + var dataviews = requestMapConfig.dataviews || {}; + + var dataviewsFiltersBySourceId = Object.keys(dataviewsFilters).reduce(function(bySourceId, dataviewName) { + var dataview = dataviews[dataviewName]; + if (dataview) { + var sourceId = dataview.source.id; + if (!bySourceId.hasOwnProperty(sourceId)) { + bySourceId[sourceId] = {}; + } + + bySourceId[sourceId][dataviewName] = { + type: 'range', + column: dataview.options.column, + params: dataviewsFilters[dataviewName] + }; + } + return bySourceId; + }, {}); + + debug(dataviewsFiltersBySourceId); debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4)); if (Array.isArray(requestMapConfig.analyses)) { requestMapConfig = replaceSourceRootQueries(requestMapConfig); + requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); } if (!shouldAdaptLayers(requestMapConfig)) { diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js new file mode 100644 index 00000000..e5a74d36 --- /dev/null +++ b/lib/cartodb/models/dataview/histogram.js @@ -0,0 +1,318 @@ +var _ = require('underscore'); +var debug = require('debug')('windshaft:dataview:histogram'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); + +var BIN_MIN_NUMBER = 6; +var BIN_MAX_NUMBER = 48; + +var basicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', + ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var overrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', + ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join('\n')); + +var iqrQueryTpl = dot.template([ + 'iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + ' FROM (', + ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', + ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', + ' ) AS quartile', + ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' WHERE quartile = 1 or quartile = 3', + ' GROUP BY quartile', + ' ) _cdb_iqr', + ')' +].join('\n')); + +var binsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + ' THEN 1', + ' ELSE GREATEST(', + ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST(', + ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' {{=it._maxBins}}', + ' )', + ' )', + ' END AS bins_number', + ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' LIMIT 1', + ')' +].join('\n')); + +var overrideBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT {{=it._bins}} AS bins_number', + ')' +].join('\n')); + +var nullsQueryTpl = dot.template([ + 'nulls AS (', + ' SELECT', + ' count(*) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' WHERE {{=it._column}} IS NULL', + ')' +].join('\n')); + +var histogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' avg_val,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' END AS bin,', + ' min({{=it._column}})::numeric AS min,', + ' max({{=it._column}})::numeric AS max,', + ' avg({{=it._column}})::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', + 'WHERE {{=it._column}} IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'ORDER BY bin' +].join('\n')); + + +var TYPE = 'histogram'; + +/** + { + type: 'histogram', + options: { + column: 'name', + bins: 10 // OPTIONAL + } + } + */ +function Histogram(query, options) { + if (!_.isString(options.column)) { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; +} + +module.exports = Histogram; + +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + + +Histogram.prototype.getResult = function(psql, override, callback) { + var self = this; + this.sql(psql, override, function(err, query) { + psql.query(query, function(err, result) { + + if (err) { + return callback(err, result); + } + + result = self.format(result, override); + result.type = self.getType(); + + return callback(null, result); + + }, true); // use read-only transaction + }); + +}; + +Histogram.prototype.search = function(psql, filters, userQuery, callback) { + return callback(null, this.format({ rows: [] })); +}; + + +Histogram.prototype.sql = function(psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var self = this; + + var _column = this.column; + + var columnTypeQuery = columnTypeQueryTpl({ + column: _column, query: this.query + }); + + if (this._columnType === null) { + psql.query(columnTypeQuery, function(err, result) { + // assume numeric, will fail later + self._columnType = 'numeric'; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (DATE_OIDS.hasOwnProperty(pgType)) { + self._columnType = 'date'; + } + } + self.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + if (this._columnType === 'date') { + _column = columnCastTpl({column: _column}); + } + + var _query = this.query; + + var basicsQuery, binsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + debug('overriding with %j', override); + basicsQuery = overrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _start: override.start, + _end: override.end + }); + + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + basicsQuery = basicsQueryTpl({ + _query: _query, + _column: _column + }); + + if (override && _.has(override, 'bins')) { + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + binsQuery = [ + iqrQueryTpl({ + _query: _query, + _column: _column + }), + binsQueryTpl({ + _query: _query, + _minBins: BIN_MIN_NUMBER, + _maxBins: BIN_MAX_NUMBER + }) + ].join(',\n'); + } + } + + + var histogramSql = [ + "WITH", + [ + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ].join(',\n'), + histogramQueryTpl({ + _query: _query, + _column: _column + }) + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + +Histogram.prototype.format = function(result, override) { + override = override || {}; + var buckets = []; + + var binsCount = getBinsCount(override); + var width = getWidth(override); + var binsStart = getBinStart(override); + var nulls = 0; + var avg; + + if (result.rows.length) { + var firstRow = result.rows[0]; + binsCount = firstRow.bins_number; + width = firstRow.bin_width || width; + avg = firstRow.avg_val; + nulls = firstRow.nulls_count; + binsStart = override.hasOwnProperty('start') ? override.start : firstRow.min; + + buckets = result.rows.map(function(row) { + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + }); + } + + return { + bin_width: width, + bins_count: binsCount, + bins_start: binsStart, + nulls: nulls, + avg: avg, + bins: buckets + }; +}; + +function getBinStart(override) { + return override.start || 0; +} + +function getBinsCount(override) { + return override.bins || 0; +} + +function getWidth(override) { + var width = 0; + var binsCount = override.bins; + + if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { + width = (override.end - override.start) / binsCount; + } + + return width; +} + +Histogram.prototype.getType = function() { + return TYPE; +}; + +Histogram.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); +}; diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js new file mode 100644 index 00000000..b7a3c249 --- /dev/null +++ b/lib/cartodb/models/filter/bbox.js @@ -0,0 +1,120 @@ +var debug = require('debug')('windshaft:filter:bbox'); +var dot = require('dot'); +dot.templateSettings.strip = false; + +var filterQueryTpl = dot.template([ + 'SELECT * FROM ({{=it._sql}}) _cdb_bbox_filter', + 'WHERE {{=it._filters}}' +].join('\n')); + +var bboxFilterTpl = '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'; + +var LATITUDE_MAX_VALUE = 85.0511287798066; +var LONGITUDE_LOWER_BOUND = -180; +var LONGITUDE_UPPER_BOUND = 180; +var LONGITUDE_RANGE = LONGITUDE_UPPER_BOUND - LONGITUDE_LOWER_BOUND; + +/** + Definition + { + "type”: "bbox", + "options": { + "column": "the_geom_webmercator", + "srid": 3857 + } + } + + Params + { + “bbox”: "west,south,east,north" + } + */ +function BBox(filterDefinition, filterParams) { + var bbox = filterParams.bbox; + + if (!bbox) { + throw new Error('BBox filter expects to have a bbox param'); + } + + var bboxElements = bbox.split(',').map(function(e) { return +e; }); + + validateBboxElements(bboxElements); + + this.column = filterDefinition.column || 'the_geom_webmercator'; + this.srid = filterDefinition.srid || 3857; + + // Latitudes must be within max extent + var south = Math.max(bboxElements[1], -LATITUDE_MAX_VALUE); + var north = Math.min(bboxElements[3], LATITUDE_MAX_VALUE); + + // Longitudes crossing 180º need another approach + var adjustedLongitudeRange = adjustLongitudeRange([bboxElements[0], bboxElements[2]]); + var west = adjustedLongitudeRange[0]; + var east = adjustedLongitudeRange[1]; + + this.bboxes = getBoundingBoxes(west, south, east, north); +} + +function getBoundingBoxes(west, south, east, north) { + var bboxes = []; + + if (east - west >= 360) { + bboxes.push([-180, south, 180, north]); + } else if (west >= -180 && east <= 180) { + bboxes.push([west, south, east, north]); + } else { + bboxes.push([west, south, 180, north]); + bboxes.push([-180, south, east % 180, north]); + } + + return bboxes; +} + +function validateBboxElements(bboxElements) { + var isNumericBbox = bboxElements + .map(function(n) { return Number.isFinite(n); }) + .reduce(function(allFinite, isFinite) { + if (!allFinite) { + return false; + } + return isFinite; + }, true); + + if (bboxElements.length !== 4 || !isNumericBbox) { + throw new Error('Invalid bbox filter, expected format="west,south,east,north"'); + } +} + +function adjustLongitudeRange(we) { + var west = we[0]; + west -= LONGITUDE_LOWER_BOUND; + west = west - (LONGITUDE_RANGE * Math.floor(west / LONGITUDE_RANGE)) + LONGITUDE_LOWER_BOUND; + + var longitudeRange = Math.min(we[1] - we[0], 360); + + return [west, west + longitudeRange]; +} + +module.exports = BBox; + +module.exports.adjustLongitudeRange = adjustLongitudeRange; +module.exports.LATITUDE_MAX_VALUE = LATITUDE_MAX_VALUE; +module.exports.LONGITUDE_MAX_VALUE = LONGITUDE_UPPER_BOUND; + + +BBox.prototype.sql = function(rawSql) { + var bboxSql = filterQueryTpl({ + _sql: rawSql, + _filters: this.bboxes.map(function(bbox) { + return bboxFilterTpl({ + _column: this.column, + _bbox: bbox.join(','), + _srid: this.srid + }); + }.bind(this)).join(' OR ') + }); + + debug(bboxSql); + + return bboxSql; +}; diff --git a/test/acceptance/analysis/analysis-layers-dataviews.js b/test/acceptance/analysis/analysis-layers-dataviews.js new file mode 100644 index 00000000..1a2307f6 --- /dev/null +++ b/test/acceptance/analysis/analysis-layers-dataviews.js @@ -0,0 +1,124 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); +var dot = require('dot'); + +describe('analysis-layers-dataviews', function() { + + var multitypeStyleTemplate = dot.template([ + "#points['mapnik::geometry_type'=1] {", + " marker-fill-opacity: {{=it._opacity}};", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: {{=it._opacity}};", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: {{=it._color}};", + " marker-allow-overlap: true;", + "}", + "#lines['mapnik::geometry_type'=2] {", + " line-color: {{=it._color}};", + " line-width: 2;", + " line-opacity: {{=it._opacity}};", + "}", + "#polygons['mapnik::geometry_type'=3] {", + " polygon-fill: {{=it._color}};", + " polygon-opacity: {{=it._opacity}};", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: {{=it._opacity}};", + "}" + ].join('\n')); + + + function cartocss(color, opacity) { + return multitypeStyleTemplate({ + _color: color || '#F11810', + _opacity: Number.isFinite(opacity) ? opacity : 1 + }); + } + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var DEFAULT_MULTITYPE_STYLE = cartocss(); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + } + ], + { + pop_max_histogram: { + source: { + id: '2570e105-7b37-40d2-bdf4-1af889598745' + }, + type: 'histogram', + options: { + column: 'pop_max' + } + } + }, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + } + ] + ); + + it('should get histogram dataview', function(done) { + var testClient = new TestClient(mapConfig, 1234); + + testClient.getDataview('pop_max_histogram', function(err, dataview) { + assert.ok(!err, err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins_start, 0); + + testClient.drain(done); + }); + }); + + it('should get a filtered histogram dataview', function(done) { + var testClient = new TestClient(mapConfig, 1234); + + var params = { + filters: { + dataviews: { + pop_max_histogram: { + min: 2e6 + } + } + } + }; + + testClient.getDataview('pop_max_histogram', params, function(err, dataview) { + assert.ok(!err, err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins_start, 2008000); + + testClient.drain(done); + }); + }); +}); diff --git a/test/support/test-client.js b/test/support/test-client.js index b79ce9e7..2122c273 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -117,6 +117,113 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { ); }; +TestClient.prototype.getDataview = function(dataviewName, params, callback) { + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + var extraParams = {}; + if (this.apiKey) { + extraParams.api_key = this.apiKey; + } + if (params && params.filters) { + extraParams.filters = JSON.stringify(params.filters); + } + + var url = '/api/v1/map'; + if (Object.keys(extraParams).length > 0) { + url += '?' + qs.stringify(extraParams); + } + + console.log(url); + + var layergroupId; + step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + var parsedBody = JSON.parse(res.body); +// var expectedWidgetURLS = { +// http: "/api/v1/map/" + parsedBody.layergroupid + "/dataview/" + dataviewName +// }; +// assert.ok(parsedBody.metadata.dataviews[dataviewName]); +// assert.ok( +// parsedBody.metadata.dataviews[dataviewName].url.http.match(expectedWidgetURLS.http) +// ); + return next(null, parsedBody.layergroupid); + } + ); + }, + function getDataviewResult(err, _layergroupId) { + assert.ifError(err); + + var next = this; + layergroupId = _layergroupId; + + var urlParams = { + own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 + }; + if (params && params.bbox) { + urlParams.bbox = params.bbox; + } + if (self.apiKey) { + urlParams.api_key = self.apiKey; + } + url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams); + + assert.response(server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + + next(null, JSON.parse(res.body)); + } + ); + }, + function finish(err, res) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + return callback(err, res); + } + ); +}; + TestClient.prototype.getTile = function(z, x, y, params, callback) { var self = this; From 57e6e49749703104392cad39ae360e064df444a7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 17:28:36 +0100 Subject: [PATCH 38/80] Another workaround to not delete analyses if there are dataviews --- lib/cartodb/models/analysis_mapconfig_adapter.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index dbc55284..8a1d0302 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -30,10 +30,11 @@ function layerQuery(query, columnNames) { return layerQueryTemplate({ _query: query, _columns: skipColumns(columnNames).join(', ') }); } -function replaceSourceRootQueries(requestMapConfig) { - var analysisSourceRootsIds = requestMapConfig.analyses.reduce(function(sourceRootsIds, analysis) { +function replaceSourceRootQueries(requestMapConfig, analysisToRemove) { + var analysisSourceRootsIds = requestMapConfig.analyses.reduce(function(sourceRootsIds, analysis, analysisIndex) { if (analysis.type === 'source' && !!analysis.id) { sourceRootsIds[analysis.id] = analysis; + analysisToRemove[analysisIndex] = true; } return sourceRootsIds; }, {}); @@ -102,11 +103,18 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4)); + var analysisToRemove = {}; if (Array.isArray(requestMapConfig.analyses)) { - requestMapConfig = replaceSourceRootQueries(requestMapConfig); + requestMapConfig = replaceSourceRootQueries(requestMapConfig, analysisToRemove); requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); } + if (Object.keys(dataviews).length === 0) { + requestMapConfig.analyses = requestMapConfig.analyses.filter(function(analysis, index) { + return !analysisToRemove.hasOwnProperty(index); + }); + } + if (!shouldAdaptLayers(requestMapConfig)) { debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); return callback(null, requestMapConfig); From 52f35d74b9487ee343227ae2f2d2de4ced4e1d34 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 17:31:28 +0100 Subject: [PATCH 39/80] Allow a higher jshint maxcomplexity --- lib/cartodb/models/analysis_mapconfig_adapter.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 8a1d0302..77556e25 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -77,6 +77,7 @@ function shouldAdaptLayers(requestMapConfig) { } AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, filters, callback) { + // jshint maxcomplexity:7 filters = filters || {}; var dataviewsFilters = filters.dataviews || {}; debug(dataviewsFilters); From 1e239658d8287e49e65f118cf393c0ea9914156d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 17:34:40 +0100 Subject: [PATCH 40/80] Just remove analysis if there are analysis --- lib/cartodb/models/analysis_mapconfig_adapter.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 77556e25..c0cb6b8f 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -108,12 +108,12 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r if (Array.isArray(requestMapConfig.analyses)) { requestMapConfig = replaceSourceRootQueries(requestMapConfig, analysisToRemove); requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); - } - if (Object.keys(dataviews).length === 0) { - requestMapConfig.analyses = requestMapConfig.analyses.filter(function(analysis, index) { - return !analysisToRemove.hasOwnProperty(index); - }); + if (Object.keys(dataviews).length === 0) { + requestMapConfig.analyses = requestMapConfig.analyses.filter(function(analysis, index) { + return !analysisToRemove.hasOwnProperty(index); + }); + } } if (!shouldAdaptLayers(requestMapConfig)) { From f745e915d380f9bb719453bedd62e4887fabb920 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 17:49:20 +0100 Subject: [PATCH 41/80] Own filter test for dataviews --- .../analysis/analysis-layers-dataviews.js | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/acceptance/analysis/analysis-layers-dataviews.js b/test/acceptance/analysis/analysis-layers-dataviews.js index 1a2307f6..a328ed09 100644 --- a/test/acceptance/analysis/analysis-layers-dataviews.js +++ b/test/acceptance/analysis/analysis-layers-dataviews.js @@ -121,4 +121,28 @@ describe('analysis-layers-dataviews', function() { testClient.drain(done); }); }); + + it('should skip the filter when sending own_filter=0 for histogram dataview', function(done) { + var testClient = new TestClient(mapConfig, 1234); + + var params = { + filters: { + dataviews: { + pop_max_histogram: { + min: 2e6 + } + } + }, + own_filter: 0 + }; + + testClient.getDataview('pop_max_histogram', params, function(err, dataview) { + assert.ok(!err, err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins_start, 0); + + testClient.drain(done); + }); + }); }); From ff147ca3bf6b0adc0f6136e790ab2ec38f9451be Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 18 Mar 2016 18:09:17 +0100 Subject: [PATCH 42/80] Add dataviews to layergroup metadata --- lib/cartodb/controllers/map.js | 13 +++++++++++++ test/support/test-client.js | 16 +++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 0b793735..60494284 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -383,6 +383,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la // TODO this should take into account several URL patterns addWidgetsUrl(username, layergroup); + addDataviewsUrls(username, layergroup, mapconfig.obj()); if (req.method === 'GET') { var ttl = global.environment.varnish.layergroupTtl || 86400; res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); @@ -401,6 +402,18 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la ); }; +function addDataviewsUrls(username, layergroup, mapConfig) { + layergroup.metadata.dataviews = layergroup.metadata.dataviews || {}; + var dataviews = mapConfig.dataviews || {}; + + Object.keys(dataviews).forEach(function(dataviewName) { + var resource = layergroup.layergroupid + '/dataview/' + dataviewName; + layergroup.metadata.dataviews[dataviewName] = { + url: getUrls(username, resource) + }; + }); +} + function addWidgetsUrl(username, layergroup) { if (layergroup.metadata && Array.isArray(layergroup.metadata.layers)) { diff --git a/test/support/test-client.js b/test/support/test-client.js index 2122c273..b83d0589 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -138,8 +138,6 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { url += '?' + qs.stringify(extraParams); } - console.log(url); - var layergroupId; step( function createLayergroup() { @@ -165,13 +163,13 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { return next(err); } var parsedBody = JSON.parse(res.body); -// var expectedWidgetURLS = { -// http: "/api/v1/map/" + parsedBody.layergroupid + "/dataview/" + dataviewName -// }; -// assert.ok(parsedBody.metadata.dataviews[dataviewName]); -// assert.ok( -// parsedBody.metadata.dataviews[dataviewName].url.http.match(expectedWidgetURLS.http) -// ); + var expectedDataviewsURLS = { + http: "/api/v1/map/" + parsedBody.layergroupid + "/dataview/" + dataviewName + }; + assert.ok(parsedBody.metadata.dataviews[dataviewName]); + assert.ok( + parsedBody.metadata.dataviews[dataviewName].url.http.match(expectedDataviewsURLS.http) + ); return next(null, parsedBody.layergroupid); } ); From d5c5c7bdbb2992fe151da2eda2ce9753afb7722a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 21 Mar 2016 18:02:19 +0100 Subject: [PATCH 43/80] Do not remove analysis, camshaft takes care of source root nodes now --- lib/cartodb/models/analysis_mapconfig_adapter.js | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index c0cb6b8f..27cacd1d 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -104,17 +104,7 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4)); - var analysisToRemove = {}; - if (Array.isArray(requestMapConfig.analyses)) { - requestMapConfig = replaceSourceRootQueries(requestMapConfig, analysisToRemove); - requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); - - if (Object.keys(dataviews).length === 0) { - requestMapConfig.analyses = requestMapConfig.analyses.filter(function(analysis, index) { - return !analysisToRemove.hasOwnProperty(index); - }); - } - } + requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); if (!shouldAdaptLayers(requestMapConfig)) { debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); From f928147559223db1e0264fdb4d862e46d6c15358 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 21 Mar 2016 18:16:54 +0100 Subject: [PATCH 44/80] Fix bbox template --- lib/cartodb/models/filter/bbox.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js index b7a3c249..8afdf905 100644 --- a/lib/cartodb/models/filter/bbox.js +++ b/lib/cartodb/models/filter/bbox.js @@ -7,7 +7,9 @@ var filterQueryTpl = dot.template([ 'WHERE {{=it._filters}}' ].join('\n')); -var bboxFilterTpl = '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'; +var bboxFilterTpl = dot.template( + '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})' +); var LATITUDE_MAX_VALUE = 85.0511287798066; var LONGITUDE_LOWER_BOUND = -180; From ebe25761d2a63f984f6323005bee53f0dfc435a7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Mar 2016 10:52:02 +0100 Subject: [PATCH 45/80] Extract variable --- lib/cartodb/backends/dataview.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 0e52aea3..fd88ac68 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -19,6 +19,8 @@ module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { var timer = new Timer(); + var dataviewName = params.dataviewName; + var mapConfig; var dataviewDefinition; step( @@ -30,9 +32,9 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param mapConfig = _mapConfig; - var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), params.dataviewName); + var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); if (!_dataviewDefinition) { - throw new Error("Dataview '" + params.dataviewName + "' does not exists"); + throw new Error("Dataview '" + dataviewName + "' does not exists"); } dataviewDefinition = _dataviewDefinition; @@ -102,7 +104,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param query = node.getQuery(); } else { var applyFilters = {}; - applyFilters[params.dataviewName] = false; + applyFilters[dataviewName] = false; query = node.getQuery(applyFilters); } From 90b92f0180e3885de71d5b39a417b1bd8eaab5fd Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Mar 2016 12:22:04 +0100 Subject: [PATCH 46/80] Adds support for category filters --- .../models/analysis_mapconfig_adapter.js | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 27cacd1d..845c9425 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -76,6 +76,20 @@ function shouldAdaptLayers(requestMapConfig) { Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; } +var DATAVIEW_TYPE_2_FILTER_TYPE = { + aggregation: 'category', + histogram: 'range' +}; +function getFilter(dataview, params) { + var type = dataview.type; + + return { + type: DATAVIEW_TYPE_2_FILTER_TYPE[type], + column: dataview.options.column, + params: params + }; +} + AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, filters, callback) { // jshint maxcomplexity:7 filters = filters || {}; @@ -91,11 +105,7 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r bySourceId[sourceId] = {}; } - bySourceId[sourceId][dataviewName] = { - type: 'range', - column: dataview.options.column, - params: dataviewsFilters[dataviewName] - }; + bySourceId[sourceId][dataviewName] = getFilter(dataview, dataviewsFilters[dataviewName]); } return bySourceId; }, {}); From 26512f648598df81a6181850d4303074832b4d2f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Mar 2016 12:22:48 +0100 Subject: [PATCH 47/80] Remove unused function --- .../models/analysis_mapconfig_adapter.js | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 845c9425..3f5f5e8c 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -30,27 +30,6 @@ function layerQuery(query, columnNames) { return layerQueryTemplate({ _query: query, _columns: skipColumns(columnNames).join(', ') }); } -function replaceSourceRootQueries(requestMapConfig, analysisToRemove) { - var analysisSourceRootsIds = requestMapConfig.analyses.reduce(function(sourceRootsIds, analysis, analysisIndex) { - if (analysis.type === 'source' && !!analysis.id) { - sourceRootsIds[analysis.id] = analysis; - analysisToRemove[analysisIndex] = true; - } - return sourceRootsIds; - }, {}); - - requestMapConfig.layers = requestMapConfig.layers.map(function(layer) { - var sourceId = layer.options && layer.options.source && layer.options.source.id; - if (sourceId && analysisSourceRootsIds.hasOwnProperty(sourceId)) { - delete layer.options.source; - layer.options.sql = analysisSourceRootsIds[sourceId].params.query; - } - return layer; - }); - - return requestMapConfig; -} - function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { var analyses = requestMapConfig.analyses || []; From e44b5eaccdb29fd987d91f6d28e4b0b0f80101e2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Mar 2016 13:10:37 +0100 Subject: [PATCH 48/80] Fix test --- test/acceptance/analysis/analysis-layers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index 2d089cfc..7187e67c 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -176,8 +176,8 @@ describe('analysis-layers', function() { testClient.getLayergroup(PERMISSION_DENIED_RESPONSE, function(err, layergroupResult) { assert.ok(!err, err); - - assert.deepEqual(layergroupResult.errors, ["permission denied for relation cdb_tablemetadata"]); + // TODO add a better error message: Your requests requires API key as it needs write permissions. + assert.deepEqual(layergroupResult.errors, ["permission denied for relation cdb_analysis_catalog"]); testClient.drain(done); }); From 2bd3e46a4db3bf647f78a9f29a73ff79ac71608d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Mar 2016 13:10:42 +0100 Subject: [PATCH 49/80] Build dataviews with factory to generalise them --- lib/cartodb/backends/dataview.js | 6 +- lib/cartodb/models/dataview/aggregation.js | 278 +++++++++++++++++++++ lib/cartodb/models/dataview/base.js | 26 ++ lib/cartodb/models/dataview/factory.js | 18 ++ lib/cartodb/models/dataview/formula.js | 104 ++++++++ lib/cartodb/models/dataview/histogram.js | 29 +-- lib/cartodb/models/dataview/index.js | 6 + lib/cartodb/models/dataview/list.js | 66 +++++ 8 files changed, 505 insertions(+), 28 deletions(-) create mode 100644 lib/cartodb/models/dataview/aggregation.js create mode 100644 lib/cartodb/models/dataview/base.js create mode 100644 lib/cartodb/models/dataview/factory.js create mode 100644 lib/cartodb/models/dataview/formula.js create mode 100644 lib/cartodb/models/dataview/index.js create mode 100644 lib/cartodb/models/dataview/list.js diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index fd88ac68..ef11fc34 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -8,7 +8,7 @@ var step = require('step'); var Timer = require('../stats/timer'); var BBoxFilter = require('../models/filter/bbox'); -var Histogram = require('../models/dataview/histogram'); +var DataviewFactory = require('../models/dataview/factory'); function DataviewBackend() { } @@ -122,8 +122,8 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {ownFilter: ownFilter} ); - var histogramDataview = new Histogram(query, dataviewDefinition.options); - histogramDataview.getResult(pg, overrideParams, this); + var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + dataview.getResult(pg, overrideParams, this); }, function returnCallback(err, result) { return callback(err, result, timer.getTimes()); diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js new file mode 100644 index 00000000..f7a49210 --- /dev/null +++ b/lib/cartodb/models/dataview/aggregation.js @@ -0,0 +1,278 @@ +var _ = require('underscore'); +var BaseWidget = require('./base'); +var debug = require('debug')('windshaft:widget:aggregation'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var summaryQueryTpl = dot.template([ + 'summary AS (', + ' SELECT', + ' count(1) AS count,', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_aggregation_nulls', + ')' +].join('\n')); + +var rankedCategoriesQueryTpl = dot.template([ + 'categories AS(', + ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', + ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', + ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' GROUP BY {{=it._column}}', + ' ORDER BY 2 DESC', + ')' +].join('\n')); + +var categoriesSummaryQueryTpl = dot.template([ + 'categories_summary AS(', + ' SELECT count(1) categories_count, max(value) max_val, min(value) min_val', + ' FROM categories', + ')' +].join('\n')); + +var rankedAggregationQueryTpl = dot.template([ + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank < {{=it._limit}}', + 'UNION ALL', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank >= {{=it._limit}}', + 'GROUP BY nulls_count, min_val, max_val, count, categories_count' +].join('\n')); + +var aggregationQueryTpl = dot.template([ + 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', + ' nulls_count, min_val, max_val, count, categories_count', + 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'ORDER BY value DESC' +].join('\n')); + +var CATEGORIES_LIMIT = 6; + +var VALID_OPERATIONS = { + count: [], + sum: ['aggregationColumn'] +}; + +var TYPE = 'aggregation'; + +/** + { + type: 'aggregation', + options: { + column: 'name', + aggregation: 'count' // it could be, e.g., sum if column is numeric + } + } + */ +function Aggregation(query, options) { + if (!_.isString(options.column)) { + throw new Error('Aggregation expects `column` in widget options'); + } + + if (!_.isString(options.aggregation)) { + throw new Error('Aggregation expects `aggregation` operation in widget options'); + } + + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error("Aggregation does not support '" + options.aggregation + "' operation"); + } + + var requiredOptions = VALID_OPERATIONS[options.aggregation]; + var missingOptions = _.difference(requiredOptions, Object.keys(options)); + if (missingOptions.length > 0) { + throw new Error( + "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',') + ); + } + + BaseWidget.apply(this); + + this.query = query; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; +} + +Aggregation.prototype = new BaseWidget(); +Aggregation.prototype.constructor = Aggregation; + +module.exports = Aggregation; + +Aggregation.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.query; + + var aggregationSql; + if (!!override.ownFilter) { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + aggregationQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql(), + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } else { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + rankedAggregationQueryTpl({ + _query: _query, + _column: this.column, + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } + + debug(aggregationSql); + + return callback(null, aggregationSql); +}; + +var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); +Aggregation.prototype.getAggregationSql = function() { + return aggregationFnQueryTpl({ + _aggregationFn: this.aggregation, + _aggregationColumn: this.aggregationColumn || 1 + }); +}; + +Aggregation.prototype.format = function(result) { + var categories = []; + var count = 0; + var nulls = 0; + var minValue = 0; + var maxValue = 0; + var categoriesCount = 0; + + + if (result.rows.length) { + var firstRow = result.rows[0]; + count = firstRow.count; + nulls = firstRow.nulls_count; + minValue = firstRow.min_val; + maxValue = firstRow.max_val; + categoriesCount = firstRow.categories_count; + + result.rows.forEach(function(row) { + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); + }); + } + + return { + count: count, + nulls: nulls, + min: minValue, + max: maxValue, + categoriesCount: categoriesCount, + categories: categories + }; +}; + +var filterCategoriesQueryTpl = dot.template([ + 'SELECT {{=it._column}} AS category, {{=it._value}} AS value', + 'FROM ({{=it._query}}) _cdb_aggregation_search', + 'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}', + 'GROUP BY {{=it._column}}' +].join('\n')); + +var searchQueryTpl = dot.template([ + 'WITH', + 'search_unfiltered AS (', + ' {{=it._searchUnfiltered}}', + '),', + 'search_filtered AS (', + ' {{=it._searchFiltered}}', + '),', + 'search_union AS (', + ' SELECT * FROM search_unfiltered', + ' UNION ALL', + ' SELECT * FROM search_filtered', + ')', + 'SELECT category, sum(value) AS value', + 'FROM search_union', + 'GROUP BY category', + 'ORDER BY value desc' +].join('\n')); + + +Aggregation.prototype.search = function(psql, userQuery, callback) { + var self = this; + + var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); + + // TODO unfiltered will be wrong as filters are already applied at this point + var query = searchQueryTpl({ + _searchUnfiltered: filterCategoriesQueryTpl({ + _query: this.query, + _column: this.column, + _value: '0', + _userQuery: _userQuery + }), + _searchFiltered: filterCategoriesQueryTpl({ + _query: this.query, + _column: this.column, + _value: 'count(1)', + _userQuery: _userQuery + }) + }); + + psql.query(query, function(err, result) { + if (err) { + return callback(err, result); + } + + return callback(null, {type: self.getType(), categories: result.rows }); + }, true); // use read-only transaction +}; + +Aggregation.prototype.getType = function() { + return TYPE; +}; + +Aggregation.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _aggregation: this.aggregation + }); +}; diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js new file mode 100644 index 00000000..b2e2f188 --- /dev/null +++ b/lib/cartodb/models/dataview/base.js @@ -0,0 +1,26 @@ +function BaseDataview() {} + +module.exports = BaseDataview; + +BaseDataview.prototype.getResult = function(psql, override, callback) { + var self = this; + this.sql(psql, override, function(err, query) { + psql.query(query, function(err, result) { + + if (err) { + return callback(err, result); + } + + result = self.format(result, override); + result.type = self.getType(); + + return callback(null, result); + + }, true); // use read-only transaction + }); + +}; + +BaseDataview.prototype.search = function(psql, userQuery, callback) { + return callback(null, this.format({ rows: [] })); +}; diff --git a/lib/cartodb/models/dataview/factory.js b/lib/cartodb/models/dataview/factory.js new file mode 100644 index 00000000..464a0d44 --- /dev/null +++ b/lib/cartodb/models/dataview/factory.js @@ -0,0 +1,18 @@ +var dataviews = require('./'); + +var DataviewFactory = { + dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) { + allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName]; + return allDataviews; + }, {}), + + getDataview: function(query, dataviewDefinition) { + var type = dataviewDefinition.type; + if (!this.dataviews[type]) { + throw new Error('Invalid dataview type: "' + type + '"'); + } + return new this.dataviews[type](query, dataviewDefinition.options); + } +}; + +module.exports = DataviewFactory; diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js new file mode 100644 index 00000000..c3c176dd --- /dev/null +++ b/lib/cartodb/models/dataview/formula.js @@ -0,0 +1,104 @@ +var _ = require('underscore'); +var BaseWidget = require('./base'); +var debug = require('debug')('windshaft:widget:formula'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var formulaQueryTpl = dot.template([ + 'SELECT', + '{{=it._operation}}({{=it._column}}) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' +].join('\n')); + +var VALID_OPERATIONS = { + count: true, + avg: true, + sum: true, + min: true, + max: true +}; + +var TYPE = 'formula'; + +/** + { + type: 'formula', + options: { + column: 'name', + operation: 'count' // count, sum, avg + } + } + */ +function Formula(query, options) { + if (!_.isString(options.operation)) { + throw new Error('Formula expects `operation` in widget options'); + } + + if (!VALID_OPERATIONS[options.operation]) { + throw new Error("Formula does not support '" + options.operation + "' operation"); + } + + if (options.operation !== 'count' && !_.isString(options.column)) { + throw new Error('Formula expects `column` in widget options'); + } + + BaseWidget.apply(this); + + this.query = query; + this.column = options.column || '1'; + this.operation = options.operation; +} + +Formula.prototype = new BaseWidget(); +Formula.prototype.constructor = Formula; + +module.exports = Formula; + +Formula.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.query; + var formulaSql = formulaQueryTpl({ + _query: _query, + _operation: this.operation, + _column: this.column + }); + + debug(formulaSql); + + return callback(null, formulaSql); +}; + +Formula.prototype.format = function(result) { + var formattedResult = { + operation: this.operation, + result: 0, + nulls: 0 + }; + + if (result.rows.length) { + formattedResult.operation = this.operation; + formattedResult.result = result.rows[0].result; + formattedResult.nulls = result.rows[0].nulls_count; + } + + return formattedResult; +}; + +Formula.prototype.getType = function() { + return TYPE; +}; + +Formula.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _operation: this.operation + }); +}; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index e5a74d36..d5d611b4 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,4 +1,5 @@ var _ = require('underscore'); +var BaseWidget = require('./base'); var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); @@ -120,6 +121,9 @@ function Histogram(query, options) { this._columnType = null; } +Histogram.prototype = new BaseWidget(); +Histogram.prototype.constructor = Histogram; + module.exports = Histogram; var DATE_OIDS = { @@ -128,31 +132,6 @@ var DATE_OIDS = { 1184: true }; - -Histogram.prototype.getResult = function(psql, override, callback) { - var self = this; - this.sql(psql, override, function(err, query) { - psql.query(query, function(err, result) { - - if (err) { - return callback(err, result); - } - - result = self.format(result, override); - result.type = self.getType(); - - return callback(null, result); - - }, true); // use read-only transaction - }); - -}; - -Histogram.prototype.search = function(psql, filters, userQuery, callback) { - return callback(null, this.format({ rows: [] })); -}; - - Histogram.prototype.sql = function(psql, override, callback) { if (!callback) { callback = override; diff --git a/lib/cartodb/models/dataview/index.js b/lib/cartodb/models/dataview/index.js new file mode 100644 index 00000000..d508f1bb --- /dev/null +++ b/lib/cartodb/models/dataview/index.js @@ -0,0 +1,6 @@ +module.exports = { + Aggregation: require('./aggregation'), + Formula: require('./formula'), + Histogram: require('./histogram'), + List: require('./list') +}; diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js new file mode 100644 index 00000000..4127103a --- /dev/null +++ b/lib/cartodb/models/dataview/list.js @@ -0,0 +1,66 @@ +var dot = require('dot'); +dot.templateSettings.strip = false; + +var BaseWidget = require('./base'); + +var TYPE = 'list'; + +var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list'); + +/** +{ + type: 'list', + options: { + columns: ['name', 'description'] + } +} +*/ + +function List(query, options) { + options = options || {}; + + if (!Array.isArray(options.columns)) { + throw new Error('List expects `columns` array in widget options'); + } + + BaseWidget.apply(this); + + this.query = query; + this.columns = options.columns; +} + +List.prototype = new BaseWidget(); +List.prototype.constructor = List; + +module.exports = List; + +List.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + } + + var listSql = listSqlTpl({ + _query: this.query, + _columns: this.columns.join(', ') + }); + + return callback(null, listSql); +}; + +List.prototype.format = function(result) { + return { + rows: result.rows + }; +}; + +List.prototype.getType = function() { + return TYPE; +}; + +List.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _columns: this.columns.join(', ') + }); +}; From 7b7e9ffe599013f36f2800d8add390f10366df23 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 23 Mar 2016 12:13:26 +0100 Subject: [PATCH 50/80] Upgrade cartodb-psql module to get some bufixes --- npm-shrinkwrap.json | 936 +++++++++++++++++++++++--------------------- package.json | 2 +- 2 files changed, 491 insertions(+), 447 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index efd63cee..02c28e64 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -90,7 +90,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.2 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -114,30 +114,6 @@ "from": "async@>=1.5.2 <2.0.0", "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" }, - "cartodb-psql": { - "version": "0.6.1", - "from": "cartodb-psql@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.6.1.tgz", - "dependencies": { - "pg": { - "version": "2.6.2-cdb3", - "from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb3", - "resolved": "git://github.com/CartoDB/node-postgres.git#069c5296d1a093077feff21719641bb9e71fc50e", - "dependencies": { - "generic-pool": { - "version": "2.0.3", - "from": "generic-pool@2.0.3", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.0.3.tgz" - }, - "buffer-writer": { - "version": "1.0.0", - "from": "buffer-writer@1.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz" - } - } - } - } - }, "request": { "version": "2.69.0", "from": "request@>=2.69.0 <3.0.0", @@ -154,9 +130,9 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz", "dependencies": { "lru-cache": { - "version": "4.0.0", + "version": "4.0.1", "from": "lru-cache@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.1.tgz", "dependencies": { "pseudomap": { "version": "1.0.2", @@ -533,14 +509,14 @@ } }, "cartodb-psql": { - "version": "0.4.0", - "from": "cartodb-psql@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.4.0.tgz", + "version": "0.6.1", + "from": "cartodb-psql@>=0.6.1 <0.7.0", + "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.6.1.tgz", "dependencies": { "pg": { - "version": "2.6.2-cdb1", - "from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb1", - "resolved": "git://github.com/CartoDB/node-postgres.git#836a2dc3131e873fc4ba20cd16e7fb69a7dca98a", + "version": "2.6.2-cdb3", + "from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb3", + "resolved": "git://github.com/CartoDB/node-postgres.git#069c5296d1a093077feff21719641bb9e71fc50e", "dependencies": { "generic-pool": { "version": "2.0.3", @@ -1035,7 +1011,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.2 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -1407,9 +1383,9 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz", "dependencies": { "lru-cache": { - "version": "4.0.0", + "version": "4.0.1", "from": "lru-cache@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.1.tgz", "dependencies": { "pseudomap": { "version": "1.0.2", @@ -1848,30 +1824,6 @@ } } }, - "cartodb-psql": { - "version": "0.6.1", - "from": "cartodb-psql@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.6.1.tgz", - "dependencies": { - "pg": { - "version": "2.6.2-cdb3", - "from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb3", - "resolved": "git://github.com/CartoDB/node-postgres.git#069c5296d1a093077feff21719641bb9e71fc50e", - "dependencies": { - "generic-pool": { - "version": "2.0.3", - "from": "generic-pool@2.0.3", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.0.3.tgz" - }, - "buffer-writer": { - "version": "1.0.0", - "from": "buffer-writer@1.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz" - } - } - } - } - }, "grainstore": { "version": "1.1.1", "from": "grainstore@1.1.1", @@ -3456,23 +3408,23 @@ } }, "sqlite3": { - "version": "3.1.1", + "version": "3.1.2", "from": "sqlite3@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-3.1.1.tgz", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-3.1.2.tgz", "dependencies": { "nan": { - "version": "2.1.0", - "from": "nan@>=2.1.0 <2.2.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.1.0.tgz" + "version": "2.2.0", + "from": "nan@>=2.2.0 <2.3.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.2.0.tgz" }, "node-pre-gyp": { - "version": "0.6.14", - "from": "node-pre-gyp@~0.6.14", + "version": "0.6.24", + "from": "node-pre-gyp@~0.6.18", "dependencies": { "nopt": { - "version": "3.0.4", + "version": "3.0.6", "from": "nopt@~3.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.4.tgz", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", "dependencies": { "abbrev": { "version": "1.0.7", @@ -3482,133 +3434,122 @@ } }, "npmlog": { - "version": "1.2.1", - "from": "npmlog@~1.2.0", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-1.2.1.tgz", + "version": "2.0.3", + "from": "npmlog@~2.0.0", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-2.0.3.tgz", "dependencies": { "ansi": { - "version": "0.3.0", - "from": "ansi@~0.3.0", - "resolved": "https://registry.npmjs.org/ansi/-/ansi-0.3.0.tgz" + "version": "0.3.1", + "from": "ansi@~0.3.1", + "resolved": "https://registry.npmjs.org/ansi/-/ansi-0.3.1.tgz" }, "are-we-there-yet": { - "version": "1.0.4", - "from": "are-we-there-yet@~1.0.0", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.0.4.tgz", + "version": "1.1.2", + "from": "are-we-there-yet@~1.1.2", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz", "dependencies": { "delegates": { - "version": "0.1.0", - "from": "delegates@^0.1.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-0.1.0.tgz" + "version": "1.0.0", + "from": "delegates@^1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz" }, "readable-stream": { - "version": "1.1.13", - "from": "readable-stream@^1.1.13", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.13.tgz", + "version": "2.0.6", + "from": "readable-stream@~2.0.5", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", "dependencies": { "core-util-is": { - "version": "1.0.1", + "version": "1.0.2", "from": "core-util-is@~1.0.0", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "inherits": { + "version": "2.0.1", + "from": "inherits@~2.0.1", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" }, "isarray": { - "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + "version": "1.0.0", + "from": "isarray@~1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + }, + "process-nextick-args": { + "version": "1.0.6", + "from": "process-nextick-args@~1.0.6", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" }, "string_decoder": { "version": "0.10.31", "from": "string_decoder@~0.10.x", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, - "inherits": { - "version": "2.0.1", - "from": "inherits@2", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + "util-deprecate": { + "version": "1.0.2", + "from": "util-deprecate@~1.0.1", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" } } } } }, "gauge": { - "version": "1.2.2", - "from": "gauge@~1.2.0", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-1.2.2.tgz", + "version": "1.2.7", + "from": "gauge@~1.2.5", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-1.2.7.tgz", "dependencies": { "has-unicode": { - "version": "1.0.0", - "from": "has-unicode@^1.0.0", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-1.0.0.tgz" + "version": "2.0.0", + "from": "has-unicode@^2.0.0", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.0.tgz" }, "lodash.pad": { - "version": "3.1.1", - "from": "lodash.pad@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-3.1.1.tgz", + "version": "4.1.0", + "from": "lodash.pad@^4.1.0", + "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-4.1.0.tgz", "dependencies": { - "lodash._basetostring": { - "version": "3.0.1", - "from": "lodash._basetostring@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.1.tgz" + "lodash.repeat": { + "version": "4.0.0", + "from": "lodash.repeat@^4.0.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-4.0.0.tgz" }, - "lodash._createpadding": { - "version": "3.6.1", - "from": "lodash._createpadding@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.1.tgz", - "dependencies": { - "lodash.repeat": { - "version": "3.0.1", - "from": "lodash.repeat@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.1.tgz" - } - } + "lodash.tostring": { + "version": "4.1.2", + "from": "lodash.tostring@^4.0.0", + "resolved": "https://registry.npmjs.org/lodash.tostring/-/lodash.tostring-4.1.2.tgz" } } }, - "lodash.padleft": { - "version": "3.1.1", - "from": "lodash.padleft@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash.padleft/-/lodash.padleft-3.1.1.tgz", + "lodash.padend": { + "version": "4.2.0", + "from": "lodash.padend@^4.1.0", + "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.2.0.tgz", "dependencies": { - "lodash._basetostring": { - "version": "3.0.1", - "from": "lodash._basetostring@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.1.tgz" + "lodash.repeat": { + "version": "4.0.0", + "from": "lodash.repeat@^4.0.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-4.0.0.tgz" }, - "lodash._createpadding": { - "version": "3.6.1", - "from": "lodash._createpadding@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.1.tgz", - "dependencies": { - "lodash.repeat": { - "version": "3.0.1", - "from": "lodash.repeat@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.1.tgz" - } - } + "lodash.tostring": { + "version": "4.1.2", + "from": "lodash.tostring@^4.0.0", + "resolved": "https://registry.npmjs.org/lodash.tostring/-/lodash.tostring-4.1.2.tgz" } } }, - "lodash.padright": { - "version": "3.1.1", - "from": "lodash.padright@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash.padright/-/lodash.padright-3.1.1.tgz", + "lodash.padstart": { + "version": "4.2.0", + "from": "lodash.padstart@^4.1.0", + "resolved": "https://registry.npmjs.org/lodash.padstart/-/lodash.padstart-4.2.0.tgz", "dependencies": { - "lodash._basetostring": { - "version": "3.0.1", - "from": "lodash._basetostring@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.1.tgz" + "lodash.repeat": { + "version": "4.0.0", + "from": "lodash.repeat@^4.0.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-4.0.0.tgz" }, - "lodash._createpadding": { - "version": "3.6.1", - "from": "lodash._createpadding@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.1.tgz", - "dependencies": { - "lodash.repeat": { - "version": "3.0.1", - "from": "lodash.repeat@^3.0.0", - "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.1.tgz" - } - } + "lodash.tostring": { + "version": "4.1.2", + "from": "lodash.tostring@^4.0.0", + "resolved": "https://registry.npmjs.org/lodash.tostring/-/lodash.tostring-4.1.2.tgz" } } } @@ -3617,24 +3558,53 @@ } }, "request": { - "version": "2.64.0", + "version": "2.69.0", "from": "request@2.x", - "resolved": "https://registry.npmjs.org/request/-/request-2.64.0.tgz", + "resolved": "https://registry.npmjs.org/request/-/request-2.69.0.tgz", "dependencies": { + "aws-sign2": { + "version": "0.6.0", + "from": "aws-sign2@~0.6.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz" + }, + "aws4": { + "version": "1.3.2", + "from": "aws4@^1.2.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz", + "dependencies": { + "lru-cache": { + "version": "4.0.0", + "from": "lru-cache@^4.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz", + "dependencies": { + "pseudomap": { + "version": "1.0.2", + "from": "pseudomap@^1.0.1", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz" + }, + "yallist": { + "version": "2.0.0", + "from": "yallist@^2.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.0.0.tgz" + } + } + } + } + }, "bl": { - "version": "1.0.0", + "version": "1.0.3", "from": "bl@~1.0.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz", "dependencies": { "readable-stream": { - "version": "2.0.2", - "from": "readable-stream@~2.0.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.2.tgz", + "version": "2.0.6", + "from": "readable-stream@~2.0.5", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", "dependencies": { "core-util-is": { - "version": "1.0.1", + "version": "1.0.2", "from": "core-util-is@~1.0.0", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" }, "inherits": { "version": "2.0.1", @@ -3642,14 +3612,14 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" }, "isarray": { - "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + "version": "1.0.0", + "from": "isarray@~1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" }, "process-nextick-args": { - "version": "1.0.3", - "from": "process-nextick-args@~1.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.3.tgz" + "version": "1.0.6", + "from": "process-nextick-args@~1.0.6", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" }, "string_decoder": { "version": "0.10.31", @@ -3657,9 +3627,9 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, "util-deprecate": { - "version": "1.0.1", + "version": "1.0.2", "from": "util-deprecate@~1.0.1", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" } } } @@ -3670,132 +3640,9 @@ "from": "caseless@~0.11.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz" }, - "extend": { - "version": "3.0.0", - "from": "extend@~3.0.0", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" - }, - "forever-agent": { - "version": "0.6.1", - "from": "forever-agent@~0.6.0", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" - }, - "form-data": { - "version": "1.0.0-rc3", - "from": "form-data@~1.0.0-rc1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc3.tgz", - "dependencies": { - "async": { - "version": "1.4.2", - "from": "async@^1.4.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.4.2.tgz" - } - } - }, - "json-stringify-safe": { - "version": "5.0.1", - "from": "json-stringify-safe@~5.0.0", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" - }, - "mime-types": { - "version": "2.1.7", - "from": "mime-types@~2.1.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz", - "dependencies": { - "mime-db": { - "version": "1.19.0", - "from": "mime-db@~1.19.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz" - } - } - }, - "node-uuid": { - "version": "1.4.3", - "from": "node-uuid@~1.4.0", - "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.3.tgz" - }, - "qs": { - "version": "5.1.0", - "from": "qs@~5.1.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz" - }, - "tunnel-agent": { - "version": "0.4.1", - "from": "tunnel-agent@~0.4.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.1.tgz" - }, - "tough-cookie": { - "version": "2.1.0", - "from": "tough-cookie@>=0.12.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.1.0.tgz" - }, - "http-signature": { - "version": "0.11.0", - "from": "http-signature@~0.11.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.11.0.tgz", - "dependencies": { - "assert-plus": { - "version": "0.1.5", - "from": "assert-plus@^0.1.5", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz" - }, - "asn1": { - "version": "0.1.11", - "from": "asn1@0.1.11", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.1.11.tgz" - }, - "ctype": { - "version": "0.5.3", - "from": "ctype@0.5.3", - "resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz" - } - } - }, - "oauth-sign": { - "version": "0.8.0", - "from": "oauth-sign@~0.8.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.0.tgz" - }, - "hawk": { - "version": "3.1.0", - "from": "hawk@~3.1.0", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.0.tgz", - "dependencies": { - "hoek": { - "version": "2.16.3", - "from": "hoek@2.x.x", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz" - }, - "boom": { - "version": "2.9.0", - "from": "boom@^2.8.x", - "resolved": "https://registry.npmjs.org/boom/-/boom-2.9.0.tgz" - }, - "cryptiles": { - "version": "2.0.5", - "from": "cryptiles@2.x.x", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz" - }, - "sntp": { - "version": "1.0.9", - "from": "sntp@1.x.x", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz" - } - } - }, - "aws-sign2": { - "version": "0.5.0", - "from": "aws-sign2@~0.5.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz" - }, - "stringstream": { - "version": "0.0.4", - "from": "stringstream@~0.0.4", - "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.4.tgz" - }, "combined-stream": { "version": "1.0.5", - "from": "combined-stream@~1.0.1", + "from": "combined-stream@~1.0.5", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", "dependencies": { "delayed-stream": { @@ -3805,35 +3652,54 @@ } } }, - "isstream": { - "version": "0.1.2", - "from": "isstream@~0.1.1", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" + "extend": { + "version": "3.0.0", + "from": "extend@~3.0.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" + }, + "forever-agent": { + "version": "0.6.1", + "from": "forever-agent@~0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" + }, + "form-data": { + "version": "1.0.0-rc4", + "from": "form-data@~1.0.0-rc3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc4.tgz", + "dependencies": { + "async": { + "version": "1.5.2", + "from": "async@^1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" + } + } }, "har-validator": { - "version": "1.8.0", - "from": "har-validator@^1.6.1", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz", + "version": "2.0.6", + "from": "har-validator@~2.0.6", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", "dependencies": { - "bluebird": { - "version": "2.10.2", - "from": "bluebird@^2.9.30", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.10.2.tgz" - }, "chalk": { "version": "1.1.1", - "from": "chalk@^1.0.0", + "from": "chalk@^1.1.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz", "dependencies": { "ansi-styles": { - "version": "2.1.0", + "version": "2.2.0", "from": "ansi-styles@^2.1.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.0.tgz", + "dependencies": { + "color-convert": { + "version": "1.0.0", + "from": "color-convert@^1.0.0", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.0.0.tgz" + } + } }, "escape-string-regexp": { - "version": "1.0.3", + "version": "1.0.5", "from": "escape-string-regexp@^1.0.2", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" }, "has-ansi": { "version": "2.0.0", @@ -3848,9 +3714,9 @@ } }, "strip-ansi": { - "version": "3.0.0", + "version": "3.0.1", "from": "strip-ansi@^3.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "dependencies": { "ansi-regex": { "version": "2.0.0", @@ -3867,9 +3733,9 @@ } }, "commander": { - "version": "2.8.1", - "from": "commander@^2.8.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", + "version": "2.9.0", + "from": "commander@^2.9.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", "dependencies": { "graceful-readlink": { "version": "1.0.1", @@ -3879,9 +3745,9 @@ } }, "is-my-json-valid": { - "version": "2.12.2", - "from": "is-my-json-valid@^2.12.0", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.2.tgz", + "version": "2.13.1", + "from": "is-my-json-valid@^2.12.4", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz", "dependencies": { "generate-function": { "version": "2.0.0", @@ -3906,20 +3772,194 @@ "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz" }, "xtend": { - "version": "4.0.0", + "version": "4.0.1", "from": "xtend@^4.0.0", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + } + } + }, + "pinkie-promise": { + "version": "2.0.0", + "from": "pinkie-promise@^2.0.0", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.0.tgz", + "dependencies": { + "pinkie": { + "version": "2.0.4", + "from": "pinkie@^2.0.0", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" } } } } + }, + "hawk": { + "version": "3.1.3", + "from": "hawk@~3.1.0", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", + "dependencies": { + "hoek": { + "version": "2.16.3", + "from": "hoek@2.x.x", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz" + }, + "boom": { + "version": "2.10.1", + "from": "boom@2.x.x", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz" + }, + "cryptiles": { + "version": "2.0.5", + "from": "cryptiles@2.x.x", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz" + }, + "sntp": { + "version": "1.0.9", + "from": "sntp@1.x.x", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz" + } + } + }, + "http-signature": { + "version": "1.1.1", + "from": "http-signature@~1.1.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", + "dependencies": { + "assert-plus": { + "version": "0.2.0", + "from": "assert-plus@^0.2.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz" + }, + "jsprim": { + "version": "1.2.2", + "from": "jsprim@^1.2.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.2.2.tgz", + "dependencies": { + "extsprintf": { + "version": "1.0.2", + "from": "extsprintf@1.0.2", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz" + }, + "json-schema": { + "version": "0.2.2", + "from": "json-schema@0.2.2", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz" + }, + "verror": { + "version": "1.3.6", + "from": "verror@1.3.6", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz" + } + } + }, + "sshpk": { + "version": "1.7.4", + "from": "sshpk@^1.7.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.7.4.tgz", + "dependencies": { + "asn1": { + "version": "0.2.3", + "from": "asn1@>=0.2.3 <0.3.0", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz" + }, + "dashdash": { + "version": "1.13.0", + "from": "dashdash@>=1.10.1 <2.0.0", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.13.0.tgz", + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@^1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } + }, + "jsbn": { + "version": "0.1.0", + "from": "jsbn@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz" + }, + "tweetnacl": { + "version": "0.14.1", + "from": "tweetnacl@>=0.13.0 <1.0.0", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.1.tgz" + }, + "jodid25519": { + "version": "1.0.2", + "from": "jodid25519@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz" + }, + "ecc-jsbn": { + "version": "0.1.1", + "from": "ecc-jsbn@>=0.0.1 <1.0.0", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz" + } + } + } + } + }, + "is-typedarray": { + "version": "1.0.0", + "from": "is-typedarray@~1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" + }, + "isstream": { + "version": "0.1.2", + "from": "isstream@~0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" + }, + "json-stringify-safe": { + "version": "5.0.1", + "from": "json-stringify-safe@~5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + }, + "mime-types": { + "version": "2.1.10", + "from": "mime-types@~2.1.7", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", + "dependencies": { + "mime-db": { + "version": "1.22.0", + "from": "mime-db@~1.22.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" + } + } + }, + "node-uuid": { + "version": "1.4.7", + "from": "node-uuid@~1.4.7", + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz" + }, + "oauth-sign": { + "version": "0.8.1", + "from": "oauth-sign@~0.8.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.1.tgz" + }, + "qs": { + "version": "6.0.2", + "from": "qs@~6.0.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.0.2.tgz" + }, + "stringstream": { + "version": "0.0.5", + "from": "stringstream@~0.0.4", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz" + }, + "tough-cookie": { + "version": "2.2.2", + "from": "tough-cookie@~2.2.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz" + }, + "tunnel-agent": { + "version": "0.4.2", + "from": "tunnel-agent@~0.4.1", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.2.tgz" } } }, "semver": { - "version": "5.0.3", - "from": "semver@~5.0.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz" + "version": "5.1.0", + "from": "semver@~5.1.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz" }, "tar": { "version": "2.2.1", @@ -3937,9 +3977,9 @@ "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.8.tgz", "dependencies": { "graceful-fs": { - "version": "4.1.2", + "version": "4.1.3", "from": "graceful-fs@^4.1.2", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.2.tgz" + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.3.tgz" } } }, @@ -3951,85 +3991,31 @@ } }, "tar-pack": { - "version": "2.0.0", - "from": "tar-pack@~2.0.0", - "resolved": "https://registry.npmjs.org/tar-pack/-/tar-pack-2.0.0.tgz", + "version": "3.1.3", + "from": "tar-pack@~3.1.0", + "resolved": "https://registry.npmjs.org/tar-pack/-/tar-pack-3.1.3.tgz", "dependencies": { - "uid-number": { - "version": "0.0.3", - "from": "uid-number@0.0.3", - "resolved": "https://registry.npmjs.org/uid-number/-/uid-number-0.0.3.tgz" - }, - "once": { - "version": "1.1.1", - "from": "once@~1.1.1", - "resolved": "https://registry.npmjs.org/once/-/once-1.1.1.tgz" - }, "debug": { - "version": "0.7.4", - "from": "debug@~0.7.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-0.7.4.tgz" - }, - "rimraf": { - "version": "2.2.8", - "from": "rimraf@~2.2.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz" + "version": "2.2.0", + "from": "debug@~2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "dependencies": { + "ms": { + "version": "0.7.1", + "from": "ms@0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" + } + } }, "fstream": { - "version": "0.1.31", - "from": "fstream@~0.1.22", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-0.1.31.tgz", + "version": "1.0.8", + "from": "fstream@^1.0.2", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.8.tgz", "dependencies": { "graceful-fs": { - "version": "3.0.8", - "from": "graceful-fs@~3.0.2", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.8.tgz" - }, - "inherits": { - "version": "2.0.1", - "from": "inherits@~2.0.0", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" - } - } - }, - "tar": { - "version": "0.1.20", - "from": "tar@~0.1.17", - "resolved": "https://registry.npmjs.org/tar/-/tar-0.1.20.tgz", - "dependencies": { - "block-stream": { - "version": "0.0.8", - "from": "block-stream@*", - "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.8.tgz" - }, - "inherits": { - "version": "2.0.1", - "from": "inherits@2", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" - } - } - }, - "fstream-ignore": { - "version": "0.0.7", - "from": "fstream-ignore@0.0.7", - "resolved": "https://registry.npmjs.org/fstream-ignore/-/fstream-ignore-0.0.7.tgz", - "dependencies": { - "minimatch": { - "version": "0.2.14", - "from": "minimatch@~0.2.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz", - "dependencies": { - "lru-cache": { - "version": "2.7.0", - "from": "lru-cache@2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.0.tgz" - }, - "sigmund": { - "version": "1.0.1", - "from": "sigmund@~1.0.0", - "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz" - } - } + "version": "4.1.3", + "from": "graceful-fs@^4.1.2", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.3.tgz" }, "inherits": { "version": "2.0.1", @@ -4038,37 +4024,95 @@ } } }, + "fstream-ignore": { + "version": "1.0.3", + "from": "fstream-ignore@~1.0.3", + "resolved": "https://registry.npmjs.org/fstream-ignore/-/fstream-ignore-1.0.3.tgz", + "dependencies": { + "inherits": { + "version": "2.0.1", + "from": "inherits@2", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + }, + "minimatch": { + "version": "3.0.0", + "from": "minimatch@^3.0.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.0.tgz", + "dependencies": { + "brace-expansion": { + "version": "1.1.3", + "from": "brace-expansion@^1.0.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.3.tgz", + "dependencies": { + "balanced-match": { + "version": "0.3.0", + "from": "balanced-match@^0.3.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.3.0.tgz" + }, + "concat-map": { + "version": "0.0.1", + "from": "concat-map@0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + } + } + } + } + } + } + }, + "once": { + "version": "1.3.3", + "from": "once@^1.3.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", + "dependencies": { + "wrappy": { + "version": "1.0.1", + "from": "wrappy@1", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" + } + } + }, "readable-stream": { - "version": "1.0.33", - "from": "readable-stream@~1.0.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.33.tgz", + "version": "2.0.6", + "from": "readable-stream@~2.0.4", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", "dependencies": { "core-util-is": { - "version": "1.0.1", + "version": "1.0.2", "from": "core-util-is@~1.0.0", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "inherits": { + "version": "2.0.1", + "from": "inherits@~2.0.1", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" }, "isarray": { - "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + "version": "1.0.0", + "from": "isarray@~1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + }, + "process-nextick-args": { + "version": "1.0.6", + "from": "process-nextick-args@~1.0.6", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" }, "string_decoder": { "version": "0.10.31", "from": "string_decoder@~0.10.x", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, - "inherits": { - "version": "2.0.1", - "from": "inherits@~2.0.1", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + "util-deprecate": { + "version": "1.0.2", + "from": "util-deprecate@~1.0.1", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" } } }, - "graceful-fs": { - "version": "1.2.3", - "from": "graceful-fs@1.2", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz" + "uid-number": { + "version": "0.0.6", + "from": "uid-number@~0.0.6", + "resolved": "https://registry.npmjs.org/uid-number/-/uid-number-0.0.6.tgz" } } }, @@ -4085,41 +4129,41 @@ } }, "rc": { - "version": "1.1.2", + "version": "1.1.6", "from": "rc@~1.1.0", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.1.6.tgz", "dependencies": { - "minimist": { - "version": "1.2.0", - "from": "minimist@^1.1.2", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz" - }, "deep-extend": { - "version": "0.2.11", - "from": "deep-extend@~0.2.5", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.2.11.tgz" - }, - "strip-json-comments": { - "version": "0.1.3", - "from": "strip-json-comments@0.1.x", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-0.1.3.tgz" + "version": "0.4.1", + "from": "deep-extend@~0.4.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.1.tgz" }, "ini": { "version": "1.3.4", "from": "ini@~1.3.0", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.4.tgz" + }, + "minimist": { + "version": "1.2.0", + "from": "minimist@^1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz" + }, + "strip-json-comments": { + "version": "1.0.4", + "from": "strip-json-comments@~1.0.4", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz" } } }, "rimraf": { - "version": "2.4.3", - "from": "rimraf@~2.4.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.3.tgz", + "version": "2.5.2", + "from": "rimraf@~2.5.0", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.5.2.tgz", "dependencies": { "glob": { - "version": "5.0.15", - "from": "glob@^5.0.14", - "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", + "version": "7.0.3", + "from": "glob@^7.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.0.3.tgz", "dependencies": { "inflight": { "version": "1.0.4", @@ -4135,7 +4179,7 @@ }, "inherits": { "version": "2.0.1", - "from": "inherits@~2.0.1", + "from": "inherits@2", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" }, "minimatch": { @@ -4144,14 +4188,14 @@ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.0.tgz", "dependencies": { "brace-expansion": { - "version": "1.1.1", + "version": "1.1.3", "from": "brace-expansion@^1.0.0", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.3.tgz", "dependencies": { "balanced-match": { - "version": "0.2.0", - "from": "balanced-match@^0.2.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.2.0.tgz" + "version": "0.3.0", + "from": "balanced-match@^0.3.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.3.0.tgz" }, "concat-map": { "version": "0.0.1", @@ -4163,9 +4207,9 @@ } }, "once": { - "version": "1.3.2", + "version": "1.3.3", "from": "once@^1.3.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.3.2.tgz", + "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", "dependencies": { "wrappy": { "version": "1.0.1", diff --git a/package.json b/package.json index 81c23080..6be80ad1 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "queue-async": "~1.0.7", "request": "~2.62.0", "cartodb-redis": "~0.13.0", - "cartodb-psql": "~0.4.0", + "cartodb-psql": "~0.6.1", "camshaft": "https://github.com/CartoDB/camshaft/tarball/master", "fastly-purge": "~1.0.1", "redis-mpool": "~0.4.0", From 499178319dbf61b099871964fb1e163dd16ee2be Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 23 Mar 2016 12:14:17 +0100 Subject: [PATCH 51/80] Add search endpoint/backend for dataviews --- lib/cartodb/backends/dataview.js | 111 +++++++++++++++++++++++++- lib/cartodb/controllers/layergroup.js | 36 ++++++++- 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index ef11fc34..67500519 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -27,7 +27,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param function getMapConfig() { mapConfigProvider.getMapConfig(this); }, - function getWidget(err, _mapConfig) { + function _getDataviewDefinition(err, _mapConfig) { assert.ifError(err); mapConfig = _mapConfig; @@ -131,6 +131,115 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ); }; +DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { + var timer = new Timer(); + + var dataviewName = params.dataviewName; + + var mapConfig; + var dataviewDefinition; + step( + function getMapConfig() { + mapConfigProvider.getMapConfig(this); + }, + function _getDataviewDefinition(err, _mapConfig) { + assert.ifError(err); + + mapConfig = _mapConfig; + + var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!_dataviewDefinition) { + throw new Error("Dataview '" + dataviewName + "' does not exists"); + } + + dataviewDefinition = _dataviewDefinition; + + return dataviewDefinition; + }, + function loadAnalysis(err) { + assert.ifError(err); + + var analysisConfiguration = { + db: { + host: params.dbhost, + port: params.dbport, + dbname: params.dbname, + user: params.dbuser, + pass: params.dbpassword + }, + batch: { + // TODO load this from configuration + endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', + username: user, + apiKey: params.api_key + } + }; + + var sourceId = dataviewDefinition.source.id; + var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId); + + var next = this; + + camshaft.create(analysisConfiguration, analysisDefinition, function(err, analysis) { + if (err) { + return next(err); + } + + var sourceId2Node = {}; + var rootNode = analysis.getRoot(); + if (rootNode.params && rootNode.params.id) { + sourceId2Node[rootNode.params.id] = rootNode; + } + + analysis.getSortedNodes().forEach(function(node) { + if (node.params && node.params.id) { + sourceId2Node[node.params.id] = node; + } + }); + + var node = sourceId2Node[sourceId]; + + if (!node) { + return next(new Error('Analysis node not found for dataview')); + } + + return next(null, node); + }); + }, + function runDataviewQuery(err, node) { + assert.ifError(err); + + var pg = new PSQL(dbParamsFromReqParams(params)); + + var ownFilter = +params.own_filter; + ownFilter = !!ownFilter; + + var query; + if (ownFilter) { + query = node.getQuery(); + } else { + var applyFilters = {}; + applyFilters[dataviewName] = false; + query = node.getQuery(applyFilters); + } + + if (params.bbox) { + var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + + var userQuery = params.q; + console.log(userQuery); + + var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + dataview.search(pg, userQuery, this); + }, + function returnCallback(err, result) { + return callback(err, result, timer.getTimes()); + } + ); +}; + function getAnalysisDefinition(mapConfigAnalyses, sourceId) { mapConfigAnalyses = mapConfigAnalyses || []; for (var i = 0; i < mapConfigAnalyses.length; i++) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index fbf8da92..b469a096 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -86,6 +86,10 @@ LayergroupController.prototype.register = function(app) { app.get(app.base_url_mapconfig + '/:token/dataview/:dataviewName', cors(), userMiddleware, this.dataview.bind(this)); + + app.get(app.base_url_mapconfig + + '/:token/dataview/:dataviewName/search', cors(), userMiddleware, + this.dataviewSearch.bind(this)); }; LayergroupController.prototype.dataview = function(req, res) { @@ -95,7 +99,7 @@ LayergroupController.prototype.dataview = function(req, res) { function setupParams() { self.req2params(req, this); }, - function retrieveList(err) { + function retrieveDataview(err) { assert.ifError(err); var mapConfigProvider = new MapStoreMapConfigProvider( @@ -107,7 +111,35 @@ LayergroupController.prototype.dataview = function(req, res) { req.profiler.add(stats || {}); if (err) { - self.sendError(req, res, err, 'GET WIDGET'); + self.sendError(req, res, err, 'GET DATAVIEW'); + } else { + self.sendResponse(req, res, tile, 200); + } + } + ); + +}; + +LayergroupController.prototype.dataviewSearch = function(req, res) { + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function searchDataview(err) { + assert.ifError(err); + + var mapConfigProvider = new MapStoreMapConfigProvider( + self.mapStore, req.context.user, self.userLimitsApi, req.params + ); + self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this); + }, + function finish(err, tile, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET DATAVIEW SEARCH'); } else { self.sendResponse(req, res, tile, 200); } From 1c250bf243fc483c90561f3e0fa860a5bf6ce867 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 7 Apr 2016 14:30:49 +0200 Subject: [PATCH 52/80] Remove dependency --- lib/cartodb/models/analysis_mapconfig_adapter.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 3f5f5e8c..46653369 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -5,8 +5,7 @@ var camshaft = require('camshaft'); var dot = require('dot'); dot.templateSettings.strip = false; -function AnalysisMapConfigAdapter(templateMaps) { - this.templateMaps = templateMaps; +function AnalysisMapConfigAdapter() { } module.exports = AnalysisMapConfigAdapter; From 077c4ab9071078a7f1e76d0bb9122f8b88cb1c62 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 7 Apr 2016 16:18:48 +0200 Subject: [PATCH 53/80] Adds analysis MapConfig adapter to named maps --- lib/cartodb/cache/named_map_provider_cache.js | 8 +- lib/cartodb/controllers/map.js | 2 + .../models/mapconfig/named_map_provider.js | 61 ++++-- lib/cartodb/server.js | 1 + test/acceptance/analysis/named-maps.js | 192 ++++++++++++++++++ test/fixtures/analysis/named-map-buffer.png | Bin 0 -> 5814 bytes 6 files changed, 251 insertions(+), 13 deletions(-) create mode 100644 test/acceptance/analysis/named-maps.js create mode 100644 test/fixtures/analysis/named-map-buffer.png diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index df536af5..80195e41 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -2,17 +2,21 @@ var _ = require('underscore'); var dot = require('dot'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); +var AnalysisMapConfigAdapter = require('../models/analysis_mapconfig_adapter'); var templateName = require('../backends/template_maps').templateName; var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, overviewsAdapter, turboCartocssAdapter) { +function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, overviewsAdapter, + turboCartocssAdapter) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(); this.overviewsAdapter = overviewsAdapter; this.turboCartocssAdapter = turboCartocssAdapter; @@ -30,10 +34,12 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok namedMapProviders[providerKey] = new NamedMapMapConfigProvider( this.templateMaps, this.pgConnection, + this.metadataBackend, this.userLimitsApi, this.namedLayersAdapter, this.overviewsAdapter, this.turboCartocssAdapter, + this.analysisMapConfigAdapter, user, templateId, config, diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 97d949df..3e73dc43 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -257,10 +257,12 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn mapConfigProvider = new NamedMapMapConfigProvider( self.templateMaps, self.pgConnection, + self.metadataBackend, self.userLimitsApi, self.namedLayersAdapter, self.overviewsAdapter, self.turboCartoCssAdapter, + self.analysisMapConfigAdapter, cdbuser, req.params.template_id, templateParams, diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index deeb39a2..b4646f7b 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -11,14 +11,16 @@ var QueryTables = require('cartodb-query-tables'); * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, - namedLayersAdapter, overviewsAdapter, turboCartoCssAdapter, +function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, + namedLayersAdapter, overviewsAdapter, turboCartoCssAdapter, analysisMapConfigAdapter, owner, templateId, config, authToken, params) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.namedLayersAdapter = namedLayersAdapter; this.turboCartoCssAdapter = turboCartoCssAdapter; + this.analysisMapConfigAdapter = analysisMapConfigAdapter; this.overviewsAdapter = overviewsAdapter; this.owner = owner; @@ -53,15 +55,29 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { var mapConfig = null; var datasource = null; var rendererParams; + var apiKey; step( function getTemplate() { self.getTemplate(this); }, - function prepareParams(err, tpl) { + function prepareDbParams(err, tpl) { + assert.ifError(err); + self.template = tpl; + + rendererParams = _.extend({}, self.params, { + user: self.owner + }); + self.setDBParams(self.owner, rendererParams, this); + }, + function getUserApiKey(err) { + assert.ifError(err); + self.metadataBackend.getUserMapKey(self.owner, this); + }, + function prepareParams(err, _apiKey) { assert.ifError(err); - self.template = tpl; + apiKey = _apiKey; var templateParams = {}; if (self.config) { @@ -78,6 +94,34 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { assert.ifError(err); return self.templateMaps.instance(self.template, templateParams); }, + function prepareAnalysisLayers(err, requestMapConfig) { + assert.ifError(err); + var analysisConfiguration = { + db: { + host: rendererParams.dbhost, + port: rendererParams.dbport, + dbname: rendererParams.dbname, + user: rendererParams.dbuser, + pass: rendererParams.dbpassword + }, + batch: { + // TODO load this from configuration + endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', + username: self.owner, + apiKey: apiKey + } + }; + + var filters = {}; + if (self.params.filters) { + try { + filters = JSON.parse(self.params.filters); + } catch (e) { + // ignore + } + } + self.analysisMapConfigAdapter.getLayers(analysisConfiguration, requestMapConfig, filters, this); + }, function prepareLayergroup(err, _mapConfig) { assert.ifError(err); var next = this; @@ -126,17 +170,10 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { return next(null, _mapConfig, datasource); }); }, - function beforeLayergroupCreate(err, _mapConfig, _datasource) { + function prepareContextLimits(err, _mapConfig, _datasource) { assert.ifError(err); mapConfig = _mapConfig; datasource = _datasource; - rendererParams = _.extend({}, self.params, { - user: self.owner - }); - self.setDBParams(self.owner, rendererParams, this); - }, - function prepareContextLimits(err) { - assert.ifError(err); self.userLimitsApi.getRenderLimits(self.owner, this); }, function cacheAndReturnMapConfig(err, renderLimits) { diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 5f628501..e04f835d 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -152,6 +152,7 @@ module.exports = function(serverOptions) { var namedMapProviderCache = new NamedMapProviderCache( templateMaps, pgConnection, + metadataBackend, userLimitsApi, overviewsAdapter, turboCartocssAdapter diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js new file mode 100644 index 00000000..17447e5c --- /dev/null +++ b/test/acceptance/analysis/named-maps.js @@ -0,0 +1,192 @@ +var assert = require('../../support/assert'); +var step = require('step'); + +//var mapnik = require('windshaft').mapnik; + +var helper = require('../../support/test_helper'); + +var CartodbWindshaft = require('../../../lib/cartodb/server'); +var serverOptions = require('../../../lib/cartodb/server_options'); +var server = new CartodbWindshaft(serverOptions); + +var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); + +describe('named-maps analysis', function() { + + var IMAGE_TOLERANCE_PER_MIL = 20; + + var username = 'localhost'; + var widgetsTemplateName = 'widgets-template'; + + var layergroupid; + var layergroup; + var keysToDelete; + + beforeEach(function(done) { + keysToDelete = {}; + + var widgetsTemplate = { + version: '0.0.1', + name: widgetsTemplateName, + layergroup: { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": '#buffer { polygon-fill: red; }', + "cartocss_version": "2.3.0" + } + } + ], + analyses: [ + { + "id": "HEAD", + "type": "buffer", + "params": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + }, + "radio": 50000 + } + } + ] + } + }; + + var template_params = {}; + + step( + function createTemplate() + { + var next = this; + assert.response( + server, + { + url: '/api/v1/map/named?api_key=1234', + method: 'POST', + headers: { + host: username, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(widgetsTemplate) + }, + { + status: 200 + }, + function(res, err) { + next(err, res); + } + ); + }, + function instantiateTemplate(err, res) { + assert.ifError(err); + + assert.deepEqual(JSON.parse(res.body), { template_id: widgetsTemplateName }); + var next = this; + assert.response( + server, + { + url: '/api/v1/map/named/' + widgetsTemplateName, + method: 'POST', + headers: { + host: username, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(template_params) + }, + { + status: 200 + }, + function(res) { + next(null, res); + } + ); + }, + function finish(err, res) { + assert.ifError(err); + + layergroup = JSON.parse(res.body); + assert.ok(layergroup.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from: " + res.body); + layergroupid = layergroup.layergroupid; + + keysToDelete['map_cfg|' + LayergroupToken.parse(layergroup.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + return done(); + } + ); + + }); + + afterEach(function(done) { + step( + function deleteTemplate(err) { + assert.ifError(err); + var next = this; + assert.response( + server, + { + url: '/api/v1/map/named/' + widgetsTemplateName + '?api_key=1234', + method: 'DELETE', + headers: { + host: username + } + }, + { + status: 204 + }, + function(res, err) { + next(err, res); + } + ); + }, + function deleteRedisKeys(err) { + assert.ifError(err); + helper.deleteRedisKeys(keysToDelete, done); + } + ); + }); + + it('should be able to retrieve widgets from all URLs', function(done) { + assert.response( + server, + { + url: '/api/v1/map/' + layergroupid + '/6/31/24.png', + method: 'GET', + encoding: 'binary', + headers: { + host: username + } + }, + { + status: 200, + headers: { + 'Content-Type': 'image/png' + } + }, + function(res, err) { + if (err) { + return done(err); + } + +// var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); +// assert.ok(image); + var fixturePath = './test/fixtures/analysis/named-map-buffer.png'; + assert.imageBufferIsSimilarToFile(res.body, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) { + assert.ok(!err, err); + done(); + }); + + } + ); + }); + +}); diff --git a/test/fixtures/analysis/named-map-buffer.png b/test/fixtures/analysis/named-map-buffer.png new file mode 100644 index 0000000000000000000000000000000000000000..6aee72e9dac394512b73271aa7b61c83f00c3843 GIT binary patch literal 5814 zcmV;n7D?%eP) zt9n#T6NRzdz;^>kBo2#dF&$FKY|4z*A>T`Eh^%FdK0%QGtLLRWO$$uMtX<+-Yy7IOOaiT%v`tDuENNnu! zOl%UM)9>r^Mo+!%LmJbdVcmL{F$&VbHu@Hzvmet3ZyS-uG-yw-ASk63FdgX0zVu*3$`Omw2HG$6da+=@MEI%ararBH~ z!|BSs6uQjMc$im1T@B=92R541KwWpg2Iz#I>T)Ybo|p{)UCjQ_Q~Wh8_m7EJg1utY zVA9Y2GV2BCcu^#SAOnSZ#GtXf^)Ay=Hf4AppwmSYZv%RyawAFa_m5mY`z^(DfF7yX zK<vj80}s#YJ<(Iu4{NBZCq=xIL>{Eq9rcO4M}Ds}$UKI4d=s3U^9ck5my z3me6cjh=g#VJVw3Tn6Z1(M6BI9$C^L(8qP3`1Nz?KP(0Y^~lV|@BzsW{I+b$aTTC* zWl;JX9&&(*jX+#tQ}6e80cZayy7aCuV!+7C|Ah|kfx+2-iqboHkpm1=`VBPhy^H1k zKAUp=4A8N%=(Tr!5d#iX^lj<%=@0l$0ZrNE-KjDuwVww$0yOC5)!Hg}Q{IaJohpmI zdDj;qRe^4AygODF@c@wY1Jv`+yHjOSH#a|oaQ{C|_ulnINL8R`9^M@*i@5wJeSv&W zb$Lmgq`JNhA}a)X&FS5_GKen)cL3DmRl?n0Kv%yq0m|*)Iv;@rv;K^cI7xNs0Ml?Z*_qKz$kU+`rSLF(@(nI_WouUppSfbcd*D}>22?-z0gBciEZ(^?*9*7_kz`lArhd34l&mX3x`mm#+!XQ~%ol9nV{` z>XAe!ob{8isjE+xZNj^J=4?v%5}MT1yYqQUW_)@4gnRu&purRM-eoie>-PA&E1jOF z`w4+A{mmyX?{b=fb@=;B1Cb@-bt&(k2rsNGzhjS}(Q|d)IyY#~0^-6_x&9S4A^K<`V}aNs958a&DG5a`~&nU9Khd8<(jw}z`e2JCbU z^bDt2d-5)CHL#wCK6t9%F;MSJ4gga?qbrjDI`_6Uu>*~l|2+)=z0dyAC|-Hnnb-n{ zL^a-JUW~`6*;o?0z1_Zoi_5E?06jkLap`SW;sUU=J+)CCD%sPeYaVxVcW+znYl%=_07(8(#}zQ4Bt zpm`!i1KoMskTN$#7c_j_lLNq$ zC)-s(828GF;)5TZfkc_Aylqttp>&7WM_6BB)`03>{> z&f8{9fI13&bTF{oNYK$|B|x`$?wy(gz!9K_G%ez@`ONnU0apMyfSwK985vcg!1&7V z3PCa_pw$ilTgpHnmA2G*NK%u%VaHz%bh{=iwRdGOm04Am)?dq@E&uOA|ABcJN%mH8t zrExxEN3Tz@Ywg2X89-!J*L*dGgaK)-ZA8en<6Fnf%Run1<-m=EIVxM(hLAs?{XSW? za&H=z@2hvG<^Vv#QD#TaaBVqEG+zxk3~(F(NNk`7tPvi^kO9c_Yyd%@0OJ5aTtK`B zF(q}JSqgR}OHylP2SOsxhQEvlfp-DQ$N_-(0X^xDoE%dQ>rBoBkZ2WPQGT*=N(ZoV zwXUgaLmONKC?5v^(jNFkUe1+Cs5W#EjJyKk06^-4*y-4g$>cl$30|+ZC^uO#@K3%n zM;{FB5jsN9s@!CSAYo3t9htn*0No0N^$xTsGg%>}p{L4;zo(Cewvii=khcO>WhN^F zxo7{a%wB4O`b2~q1gy$SRt7Xjf_sBrDANa9peun0F40ZOOI8AO`S0DBYv`_ltt2R_ zMPABNl9kArby>+uK*B^ACuE*T1Ug_^=R-IqIgm7SgC;8k3A5pz(0N0<$+2b7blR%&w&_05)q!MHk~oXErIVkRzm{_Z zX00YZAZPH;kl2|B83PhTyv-1=$p{$(^cgSj4$A?65D%bxQ|}I2Ro*rwBn{~A7T~A1 zO$kXwfKFRex8C+7Bo&}1AR+4oet|6(y70CqA*n!L(SW=GAY=+)J^(`24SZkzt)+L~ zb|fSvP#<;g4!l$hq9J4s&}~5G+khKy8xoQlr=Zw1I;aOqJVLjHnWuLCzd`QPdImM;g(pkngsF(HGLMt&Eda}UYvIj|#kk>n{Y zpHGuIc-69ug!}=4-@_}C{P@i>|3H`KUS+*%RZc?wg1E^-G47)Ym+Yp!#Tj=xim*4vxK^0E~n z2PuPrTpeDNnOr)y93IMbngq*iqvGVBNyOYs?{5~M+Zy){%>f`FrBOfkoSld*$QA#< z0l)(Mi{<`MW~v`Y41Zr{HF*nOmcESZP7D%1j^b^*^kDh^bt?(n5PxyTbteP4PX0Ln z)Sz^Ht3w}I88AOx5;k@9*}Efi0B8hk5;+``IFxXE5Evv+iSBK`RBWvE5;EAGtQp_Knns@@(p;EXin+q;)@_#u&ocm zp)H@>@>r(!;sD@DX$aBocNxxU0CZ-*K42K1v+5cAyUoeu!_NW0 z8u;(|htxEoWp9*@f4SC<&jH{B&?q(c${8U(W;p}sZBB#?Meg#hiw8jW+|Nn{$sx?E z(ivs6Zk5eNek)QQw;?g@^8}FCYp=TjpmcVL=#pG*V@Bj{*r!yY$w3(6ocNE`R|`Az z26S?XdZx3U0RaCxeT|1B?#iE4Wpx0Z{WI;Ugbq96TW|+}MrwN3O;}kQUk>m*4%8-l z%pu?2e_uQc*aP|&pLe~4mZkB%pNW=%XLM{m=Dp1xP4o~fAn$ojQ&}1R!n0`Ygb6x5WMTM-2QDW&2Kw12eyU!f2|+&ty18l_K5|Donmrbpfyev%YH9Vi z0ci8+NEfPU^q1a+8IbG;4`$av@{rCwtvZ7c^0;w_P%Wd&flCJ8YB}h8G(!gXu9{kv zYcL+VkBcI1LBpz6C1i@BzS{ftv?-mBXzSA%F1_C;}i;qSP*Kz8hZfjIf#)|p4^ZmU=hrM51sR4k__FT@}YS|kU$T|D} zEj;>k7P%U(xwwFdI$CM@RuQ?q=j0`y+W_BDTWPld&_OxFVRCjYPb35SSw~gMLqNn& z!PbkeYgHKcXE7Dxx$G%`Zo91{4pMa-#gKdX-?LIV0wG@7^@sugVrxaV&nSzXl+5r41LP(G=eK2Ytp} zSnFsK8&{yuQ+peSQl-cP2wwZm)xjh-u7I}yQ518AH~`zsBE@Q zo~r)rr|`+ndDMn5qPt^zz1-tBjn(nh>( z)|0;0uAoCdGTOTg^a*IGR4Ko&tQUwHcEE@D1ZrMYpU)f zaN>tXFEv5OpjL_*9y$P+K&H0p-2|NYkx6@v&A)#Uz%WC~z7$R60JH^~kKWsMJ*i$Y zkbxhXG~*z)bC3*BnbT0Gee@Fy1+J}n5ugJo6Jj_CJqn&c?zGgS zeKev0<^!yy+u9W~^n;VP0Xu)TWHI$Pc$42enk5&x;}}w&1#G_ic>vq>pt@JVBYtqv zy<@5PmdigmaUNVQBlm&e&6qMq1H28Wp}L7dM*isHLkH-rYZ(?Qo#MsBZ_1pf?gx?$ z0^p;vHB|TIpVmK$`U8P<;#m<=(DQ)IvFh^eseA*#>_AKO-J(Lq{BZT;lhCUndG$m( z@RV}``FCY0`MO5qFW-^%P*9Z0Js7+aT%t+Qx??M3%n#R3MfCVP@=?)kToACPK)xfJB~4@kFKgRZzT+uTyNF9MDHh!hV{)&Tjh zsnWe0um!mfVJ&3;^7Dv~sPB9b81!*PmC8A27ie}VZV6gBd5!pxl))!+K_C9=DN~2- z0v(7g4+!NNQdWBr1HAV;Foo>c=$ZRcx!BpRYyYl@f53->*KR=x7t$mc|zz5CF1&|Ts%ocP^ zOJ`qkh^ieUA*+R>c^W2so2(3EH6jD{uIsq`udGncFvE2mI>>lsF7kQuZQ!HYEky!= zzOvTLsynnfg!>$|ZQNLtnuuqXM{~dwuV3l}<4!o0A97a|KxyOue4GDeW*` z84rA{6fW$DyaoRficDR$N@KgmfrKJ6{;bFv%picM1+E3duBlgQiQ+3jR193ebR~2{S&YtnjGylv5y1ci1 zWmfk8Jskob9mo;e-=Uos9C0l>;J>_@Bvs%-jl#HQH>)X?Wq!~p1(UJns@o;vm>qPMU6 zRN2SFlUSgyt2i$`3cNwyI0xPetOYP5ObvZIU{{&_y*&MvDs6@%G6BT!Nb5n5a>#XH zTb>2<4D_lmNd_Oj z1GLl2l^zp4+DZn}N6s@)4@&QEt08&^W&+Ia+FA$wIba*p-csq_J6I*q8UIs&2D2u8 zK2aaNxKW4nRwV`KUbC0kCX*yU>Zsp5`dNk9y^lwh0pDbbYnB}Doy@k2ehyiWu5ySY z&?SHE?6K514ZhhB;2Ue{Ym4?wCIF2q&wj;}QDXDKq@T$D&f%_pwrJC2B+!|koxiv?Eyx>IY9+}GYc(Ls?7b+5Jo-s>PCo!|podDmEGcVgn^9N-nk1`mjeL7@cQ5YK zi``$*&uT4X=BiZz#_J!_qpLXRs$fd|gAA&&zn9fjRiAfg@RhfXYKVzJJOO9_V;XxM zCo2Srk2o)qD^$5jG0lF9KCS|PS=xw<0G$1gDf}CX$nOpuGG+a0Kp!#XRtg`ciu@Ut z2R$B%A?PQjyt=&WW(?5nhr6w2N+upY65_>A<`Jz5b3VjdKe0 zz|q@YQzciboc%pPAE(GxN|mJs`CdQf-M=ya13r2{*w4n8>i_@%07*qoM6N<$f~1Z= ArvLx| literal 0 HcmV?d00001 From 0981ccd0c40dd647387bd4868a5d54bbdcddd132 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 7 Apr 2016 17:58:12 +0200 Subject: [PATCH 54/80] Add metadata information about analyses --- lib/cartodb/controllers/map.js | 35 ++++++++++++++++--- .../models/analysis_mapconfig_adapter.js | 2 +- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 3e73dc43..f79ef2f5 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -130,6 +130,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { var self = this; var mapConfig; + var analysesResults = []; step( function setupParams(){ @@ -164,9 +165,10 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { } self.analysisMapConfigAdapter.getLayers(analysisConfiguration, requestMapConfig, filters, this); }, - function beforeLayergroupCreate(err, requestMapConfig) { + function beforeLayergroupCreate(err, requestMapConfig, _analysesResults) { assert.ifError(err); var next = this; + analysesResults = _analysesResults; self.namedLayersAdapter.getLayers(req.context.user, requestMapConfig.layers, self.pgConnection, function(err, layers, datasource) { if (err) { @@ -222,7 +224,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, this); + self.afterLayergroupCreate(req, res, mapConfig, analysesResults, layergroup, this); }, function finish(err, layergroup) { if (err) { @@ -282,7 +284,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, this); + self.afterLayergroupCreate(req, res, mapConfig, [],layergroup, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -303,7 +305,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }; -MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, callback) { +MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, analysesResults, layergroup, callback) { var self = this; var username = req.context.user; @@ -369,6 +371,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la // TODO this should take into account several URL patterns addWidgetsUrl(username, layergroup); addDataviewsUrls(username, layergroup, mapconfig.obj()); + addAnalysesMetadata(username, layergroup, analysesResults); if (req.method === 'GET') { var ttl = global.environment.varnish.layergroupTtl || 86400; res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); @@ -387,6 +390,30 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la ); }; +function addAnalysesMetadata(username, layergroup, analysesResults) { + analysesResults = analysesResults || []; + layergroup.metadata.analyses = layergroup.metadata.analyses || []; + + analysesResults.forEach(function(analysis, index) { + var resource = layergroup.layergroupid + '/analysis/' + index; + var nodes = [analysis.getRoot()].concat(analysis.getSortedNodes()); + layergroup.metadata.analyses.push({ + url: getUrls(username, resource), + nodes: nodes.reduce(function(nodesIdMap, node) { + if (node.params.id) { + var nodeResource = layergroup.layergroupid + '/analysis/' + index + '/node/' + node.params.id; + nodesIdMap[node.params.id] = { + query: node.getQuery(), + url: getUrls(username, nodeResource) + }; + } + + return nodesIdMap; + }, {}) + }); + }); +} + function addDataviewsUrls(username, layergroup, mapConfig) { layergroup.metadata.dataviews = layergroup.metadata.dataviews || {}; var dataviews = mapConfig.dataviews || {}; diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 46653369..09194872 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -138,6 +138,6 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); - return callback(null, requestMapConfig); + return callback(null, requestMapConfig, analysesResults); }); }; From 1d860fd202c6df49c67244cda7f20d6d3d3e15f2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 11 Apr 2016 18:49:22 +0200 Subject: [PATCH 55/80] Remove top level url for analysis --- lib/cartodb/controllers/map.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f79ef2f5..e70a11b3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -395,10 +395,8 @@ function addAnalysesMetadata(username, layergroup, analysesResults) { layergroup.metadata.analyses = layergroup.metadata.analyses || []; analysesResults.forEach(function(analysis, index) { - var resource = layergroup.layergroupid + '/analysis/' + index; var nodes = [analysis.getRoot()].concat(analysis.getSortedNodes()); layergroup.metadata.analyses.push({ - url: getUrls(username, resource), nodes: nodes.reduce(function(nodesIdMap, node) { if (node.params.id) { var nodeResource = layergroup.layergroupid + '/analysis/' + index + '/node/' + node.params.id; From dd36877a2091882291703a457b1ef273e9cf8971 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 11 Apr 2016 18:49:43 +0200 Subject: [PATCH 56/80] Add per node status --- lib/cartodb/controllers/map.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index e70a11b3..1c55a0cd 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -401,6 +401,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults) { if (node.params.id) { var nodeResource = layergroup.layergroupid + '/analysis/' + index + '/node/' + node.params.id; nodesIdMap[node.params.id] = { + status: node.getStatus(), query: node.getQuery(), url: getUrls(username, nodeResource) }; From ca564bfaadae8d5f3d8743d31a186f67f5f4151a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 11 Apr 2016 18:49:56 +0200 Subject: [PATCH 57/80] Add fake analysis node status endpoint --- lib/cartodb/controllers/layergroup.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b469a096..d00e7076 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -90,6 +90,17 @@ LayergroupController.prototype.register = function(app) { app.get(app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', cors(), userMiddleware, this.dataviewSearch.bind(this)); + + app.get(app.base_url_mapconfig + + '/:token/analysis/:analysisIndex/node/:nodeId', cors(), userMiddleware, + this.analysisNodeStatus.bind(this)); +}; + +LayergroupController.prototype.analysisNodeStatus = function(req, res) { + this.sendResponse(req, res, { + node_id: req.params.nodeId, + status: 'ready' + }, 200); }; LayergroupController.prototype.dataview = function(req, res) { From 9ff661480b5593e093e99c024e75d3fd6876a780 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 13 Apr 2016 18:02:28 +0200 Subject: [PATCH 58/80] Do not append root as it is already included in sorted nodes --- lib/cartodb/controllers/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 1c55a0cd..b97a4ab9 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -395,7 +395,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults) { layergroup.metadata.analyses = layergroup.metadata.analyses || []; analysesResults.forEach(function(analysis, index) { - var nodes = [analysis.getRoot()].concat(analysis.getSortedNodes()); + var nodes = analysis.getSortedNodes(); layergroup.metadata.analyses.push({ nodes: nodes.reduce(function(nodesIdMap, node) { if (node.params.id) { From 7a87b8ebef13a6b857507bc799ea6ca6fb2a391a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 13 Apr 2016 18:04:49 +0200 Subject: [PATCH 59/80] Use node id instead of param id for endpoint It will be easier to retrieve node status with that id --- lib/cartodb/controllers/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b97a4ab9..643c946d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -399,7 +399,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults) { layergroup.metadata.analyses.push({ nodes: nodes.reduce(function(nodesIdMap, node) { if (node.params.id) { - var nodeResource = layergroup.layergroupid + '/analysis/' + index + '/node/' + node.params.id; + var nodeResource = layergroup.layergroupid + '/analysis/' + index + '/node/' + node.id(); nodesIdMap[node.params.id] = { status: node.getStatus(), query: node.getQuery(), From 28400f4544933290ccd9a7e65951db01f82103b3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 13 Apr 2016 19:15:06 +0200 Subject: [PATCH 60/80] Better naming --- lib/cartodb/controllers/layergroup.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index d00e7076..fc2ba0e1 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -118,13 +118,13 @@ LayergroupController.prototype.dataview = function(req, res) { ); self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this); }, - function finish(err, tile, stats) { + function finish(err, dataview, stats) { req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'GET DATAVIEW'); } else { - self.sendResponse(req, res, tile, 200); + self.sendResponse(req, res, dataview, 200); } } ); @@ -146,13 +146,13 @@ LayergroupController.prototype.dataviewSearch = function(req, res) { ); self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this); }, - function finish(err, tile, stats) { + function finish(err, searchResult, stats) { req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'GET DATAVIEW SEARCH'); } else { - self.sendResponse(req, res, tile, 200); + self.sendResponse(req, res, searchResult, 200); } } ); @@ -174,13 +174,13 @@ LayergroupController.prototype.widget = function(req, res) { ); self.widgetBackend.getWidget(mapConfigProvider, req.params, this); }, - function finish(err, tile, stats) { + function finish(err, widget, stats) { req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'GET WIDGET'); } else { - self.sendResponse(req, res, tile, 200); + self.sendResponse(req, res, widget, 200); } } ); @@ -202,13 +202,13 @@ LayergroupController.prototype.widgetSearch = function(req, res) { ); self.widgetBackend.search(mapConfigProvider, req.params, this); }, - function finish(err, tile, stats) { + function finish(err, searchResult, stats) { req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'GET WIDGET'); } else { - self.sendResponse(req, res, tile, 200); + self.sendResponse(req, res, searchResult, 200); } } ); From 09687b3811b0da498b2c6644f46e8ffec71f2008 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 10:59:51 +0200 Subject: [PATCH 61/80] Proper endpoint to check node status from analysis --- lib/cartodb/backends/analysis.js | 49 +++++++++++++++++++++++++++ lib/cartodb/controllers/layergroup.js | 26 +++++++++++--- 2 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 lib/cartodb/backends/analysis.js diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js new file mode 100644 index 00000000..67a80ed3 --- /dev/null +++ b/lib/cartodb/backends/analysis.js @@ -0,0 +1,49 @@ +var PSQL = require('cartodb-psql'); + +function AnalysisBackend() { +} + +module.exports = AnalysisBackend; + + +AnalysisBackend.prototype.getNodeStatus = function (params, callback) { + var nodeId = params.nodeId; + + var statusQuery = 'SELECT node_id, status, updated_at FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\''; + + var pg = new PSQL(dbParamsFromReqParams(params)); + pg.query(statusQuery, function(err, result) { + if (err) { + return callback(err, result); + } + + result = result || {}; + + var rows = result.rows || []; + + return callback(null, rows[0] || { + node_id: nodeId, + status: 'unknown' + }); + }, true); // use read-only transaction +}; + +function dbParamsFromReqParams(params) { + var dbParams = {}; + if ( params.dbuser ) { + dbParams.user = params.dbuser; + } + if ( params.dbpassword ) { + dbParams.pass = params.dbpassword; + } + if ( params.dbhost ) { + dbParams.host = params.dbhost; + } + if ( params.dbport ) { + dbParams.port = params.dbport; + } + if ( params.dbname ) { + dbParams.dbname = params.dbname; + } + return dbParams; +} \ No newline at end of file diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index fc2ba0e1..b6289619 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -8,6 +8,7 @@ var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var DataviewBackend = require('../backends/dataview'); +var AnalysisBackend = require('../backends/analysis'); var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); @@ -41,6 +42,7 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev this.layergroupAffectedTables = layergroupAffectedTables; this.dataviewBackend = new DataviewBackend(); + this.analysisBackend = new AnalysisBackend(); } util.inherits(LayergroupController, BaseController); @@ -97,10 +99,26 @@ LayergroupController.prototype.register = function(app) { }; LayergroupController.prototype.analysisNodeStatus = function(req, res) { - this.sendResponse(req, res, { - node_id: req.params.nodeId, - status: 'ready' - }, 200); + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function retrieveNodeStatus(err) { + assert.ifError(err); + self.analysisBackend.getNodeStatus(req.params, this); + }, + function finish(err, nodeStatus, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET NODE STATUS'); + } else { + self.sendResponse(req, res, nodeStatus, 200); + } + } + ); }; LayergroupController.prototype.dataview = function(req, res) { From e037c8c1b27a239e04eaee97dfe10091ef9542cb Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 11:08:39 +0200 Subject: [PATCH 62/80] Do not use layer index as analysis node is are unique --- lib/cartodb/controllers/layergroup.js | 2 +- lib/cartodb/controllers/map.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b6289619..915ff839 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -94,7 +94,7 @@ LayergroupController.prototype.register = function(app) { this.dataviewSearch.bind(this)); app.get(app.base_url_mapconfig + - '/:token/analysis/:analysisIndex/node/:nodeId', cors(), userMiddleware, + '/:token/analysis/node/:nodeId', cors(), userMiddleware, this.analysisNodeStatus.bind(this)); }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 643c946d..811122e3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -394,12 +394,12 @@ function addAnalysesMetadata(username, layergroup, analysesResults) { analysesResults = analysesResults || []; layergroup.metadata.analyses = layergroup.metadata.analyses || []; - analysesResults.forEach(function(analysis, index) { + analysesResults.forEach(function(analysis) { var nodes = analysis.getSortedNodes(); layergroup.metadata.analyses.push({ nodes: nodes.reduce(function(nodesIdMap, node) { if (node.params.id) { - var nodeResource = layergroup.layergroupid + '/analysis/' + index + '/node/' + node.id(); + var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); nodesIdMap[node.params.id] = { status: node.getStatus(), query: node.getQuery(), From 2d4fd62acff2180eb97c1b9d29193297b568b156 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 11:44:17 +0200 Subject: [PATCH 63/80] Do not create triggers for tests --- test/support/sql/windshaft.test.sql | 65 ----------------------------- 1 file changed, 65 deletions(-) diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 7ad07821..506c12b8 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -611,71 +611,6 @@ CREATE INDEX analysis_rent_listings_the_geom_idx ON analysis_rent_listings USING CREATE INDEX analysis_rent_listings_the_geom_webmercator_idx ON analysis_rent_listings USING gist (the_geom_webmercator); - --- --- TOC entry 5623 (class 2620 OID 13870248) --- Name: test_quota; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON analysis_rent_listings FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_checkquota('0.1', '-1', 'public'); - - --- --- TOC entry 5627 (class 2620 OID 13870265) --- Name: test_quota; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON analysis_banks FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_checkquota('0.1', '-1', 'public'); - - --- --- TOC entry 5624 (class 2620 OID 13870249) --- Name: test_quota_per_row; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON analysis_rent_listings FOR EACH ROW EXECUTE PROCEDURE cartodb.cdb_checkquota('0.001', '-1', 'public'); - - --- --- TOC entry 5628 (class 2620 OID 13870266) --- Name: test_quota_per_row; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON analysis_banks FOR EACH ROW EXECUTE PROCEDURE cartodb.cdb_checkquota('0.001', '-1', 'public'); - - --- --- TOC entry 5621 (class 2620 OID 13870246) --- Name: track_updates; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON analysis_rent_listings FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); - - --- --- TOC entry 5625 (class 2620 OID 13870263) --- Name: track_updates; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON analysis_banks FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); - - --- --- TOC entry 5622 (class 2620 OID 13870247) --- Name: update_the_geom_webmercator_trigger; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON analysis_rent_listings FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); - - --- --- TOC entry 5626 (class 2620 OID 13870264) --- Name: update_the_geom_webmercator_trigger; Type: TRIGGER; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 --- - -CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON analysis_banks FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); - - -- -- TOC entry 5783 (class 0 OID 0) -- Dependencies: 804 From 26149e7755825e204ee1d382bb9e0c0385a6dcb5 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 11:53:48 +0200 Subject: [PATCH 64/80] cdb_analysis_catalog is already retrieved from camshaft --- test/support/sql/windshaft.test.sql | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 506c12b8..4690f081 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -336,22 +336,6 @@ INSERT INTO _vovw_1_test_table_overviews VALUES -- analysis tables ----------------------------------------------- - - -drop table if exists cdb_analysis_catalog; -create table cdb_analysis_catalog ( - node_id char(40) CONSTRAINT cdb_analysis_catalog_pkey PRIMARY KEY, - analysis_def json NOT NULL, - input_nodes char(40) ARRAY NOT NULL DEFAULT '{}', - affected_tables regclass[] NOT NULL DEFAULT '{}', - cache_tables regclass[] NOT NULL DEFAULT '{}', - created_at timestamp with time zone NOT NULL DEFAULT now(), - updated_at timestamp with time zone NOT NULL DEFAULT now(), - used_at timestamp with time zone NOT NULL DEFAULT now(), - hits NUMERIC DEFAULT 0, - last_used_from char(40) -); - ALTER TABLE cdb_analysis_catalog OWNER TO :TESTUSER; From 9ab4eb58016d769f502bd2f79be4fa395e877dec Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 12:56:20 +0200 Subject: [PATCH 65/80] Change error expectation --- test/acceptance/analysis/analysis-layers.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index 7187e67c..ffcc5fa3 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -176,8 +176,10 @@ describe('analysis-layers', function() { testClient.getLayergroup(PERMISSION_DENIED_RESPONSE, function(err, layergroupResult) { assert.ok(!err, err); - // TODO add a better error message: Your requests requires API key as it needs write permissions. - assert.deepEqual(layergroupResult.errors, ["permission denied for relation cdb_analysis_catalog"]); + assert.deepEqual( + layergroupResult.errors, + ["Analysis requires authentication with API key: permission denied."] + ); testClient.drain(done); }); From a9ca453b17b1e2c07e7e6e2568b52a31fe1db1a2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 13:20:22 +0200 Subject: [PATCH 66/80] Remove some JSON.stringify --- lib/cartodb/models/analysis_mapconfig_adapter.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 09194872..1fcab837 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -32,8 +32,6 @@ function layerQuery(query, columnNames) { function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { var analyses = requestMapConfig.analyses || []; - debug(JSON.stringify(requestMapConfig, null, 4)); - requestMapConfig.analyses = analyses.map(function(analysisDefinition) { var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition); var definition = analysisDefinition; @@ -44,8 +42,6 @@ function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { return definition; }); - debug(JSON.stringify(requestMapConfig, null, 4)); - return requestMapConfig; } @@ -95,7 +91,6 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); if (!shouldAdaptLayers(requestMapConfig)) { - debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); return callback(null, requestMapConfig); } From aa36236ed2994244c1287ef068d0556810f97da6 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 13:25:56 +0200 Subject: [PATCH 67/80] Rename analysis to analysis status backend, making room for analysis backend --- lib/cartodb/backends/{analysis.js => analysis-status.js} | 6 +++--- lib/cartodb/controllers/layergroup.js | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) rename lib/cartodb/backends/{analysis.js => analysis-status.js} (87%) diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis-status.js similarity index 87% rename from lib/cartodb/backends/analysis.js rename to lib/cartodb/backends/analysis-status.js index 67a80ed3..8ba7ae9d 100644 --- a/lib/cartodb/backends/analysis.js +++ b/lib/cartodb/backends/analysis-status.js @@ -1,12 +1,12 @@ var PSQL = require('cartodb-psql'); -function AnalysisBackend() { +function AnalysisStatusBackend() { } -module.exports = AnalysisBackend; +module.exports = AnalysisStatusBackend; -AnalysisBackend.prototype.getNodeStatus = function (params, callback) { +AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { var nodeId = params.nodeId; var statusQuery = 'SELECT node_id, status, updated_at FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\''; diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 915ff839..642e2a09 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -8,7 +8,7 @@ var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var DataviewBackend = require('../backends/dataview'); -var AnalysisBackend = require('../backends/analysis'); +var AnalysisStatusBackend = require('../backends/analysis-status'); var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); @@ -42,7 +42,7 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev this.layergroupAffectedTables = layergroupAffectedTables; this.dataviewBackend = new DataviewBackend(); - this.analysisBackend = new AnalysisBackend(); + this.analysisStatusBackend = new AnalysisStatusBackend(); } util.inherits(LayergroupController, BaseController); @@ -107,7 +107,7 @@ LayergroupController.prototype.analysisNodeStatus = function(req, res) { }, function retrieveNodeStatus(err) { assert.ifError(err); - self.analysisBackend.getNodeStatus(req.params, this); + self.analysisStatusBackend.getNodeStatus(req.params, this); }, function finish(err, nodeStatus, stats) { req.profiler.add(stats || {}); From 25a61c8479723dc18a92e27ffac96a8600a8b8e4 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 17:08:23 +0200 Subject: [PATCH 68/80] Remove console log --- lib/cartodb/backends/dataview.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 67500519..f9fba44f 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -229,7 +229,6 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca } var userQuery = params.q; - console.log(userQuery); var dataview = DataviewFactory.getDataview(query, dataviewDefinition); dataview.search(pg, userQuery, this); From a26025b259c2f707846348789bb9e1093ee5750c Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 17:09:07 +0200 Subject: [PATCH 69/80] Add analysis backend so it's possible to inject configuration --- lib/cartodb/backends/analysis.js | 16 +++++++++++++ lib/cartodb/backends/dataview.js | 11 ++++++--- lib/cartodb/controllers/layergroup.js | 5 ++-- lib/cartodb/controllers/map.js | 8 ++++--- .../models/analysis_mapconfig_adapter.js | 11 +++++---- lib/cartodb/server.js | 9 +++++-- lib/cartodb/server_options.js | 12 ++++++++++ test/acceptance/analysis/analysis-layers.js | 2 +- test/support/test-client.js | 24 ++++++++++++++++++- 9 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 lib/cartodb/backends/analysis.js diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js new file mode 100644 index 00000000..2133e772 --- /dev/null +++ b/lib/cartodb/backends/analysis.js @@ -0,0 +1,16 @@ +var camshaft = require('camshaft'); + +function AnalysisBackend(options) { + var batchConfig = options.batch || {}; + this.batchEndpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job'; + + var databaseService = batchConfig.databaseService || null; + this.analysisFactory = (databaseService === null) ? camshaft : new camshaft(databaseService); +} + +module.exports = AnalysisBackend; + +AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) { + analysisConfiguration.batch.endpoint = this.batchEndpoint; + this.analysisFactory.create(analysisConfiguration, analysisDefinition, callback); +}; diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index f9fba44f..f07ed6d1 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -10,13 +10,16 @@ var Timer = require('../stats/timer'); var BBoxFilter = require('../models/filter/bbox'); var DataviewFactory = require('../models/dataview/factory'); -function DataviewBackend() { +function DataviewBackend(analysisBackend) { + this.analysisBackend = analysisBackend; } module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { + var self = this; + var timer = new Timer(); var dataviewName = params.dataviewName; @@ -65,7 +68,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param var next = this; - camshaft.create(analysisConfiguration, analysisDefinition, function(err, analysis) { + self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) { if (err) { return next(err); } @@ -132,6 +135,8 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param }; DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { + var self = this; + var timer = new Timer(); var dataviewName = params.dataviewName; @@ -180,7 +185,7 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca var next = this; - camshaft.create(analysisConfiguration, analysisDefinition, function(err, analysis) { + self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) { if (err) { return next(err); } diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 642e2a09..99eca0c5 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -25,10 +25,11 @@ var QueryTables = require('cartodb-query-tables'); * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {AnalysisBackend} analysisBackend * @constructor */ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, - widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) { + widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) { BaseController.call(this, authApi, pgConnection); this.pgConnection = pgConnection; @@ -41,7 +42,7 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; - this.dataviewBackend = new DataviewBackend(); + this.dataviewBackend = new DataviewBackend(analysisBackend); this.analysisStatusBackend = new AnalysisStatusBackend(); } diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 811122e3..32c5203a 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -26,15 +26,17 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye * @param {TemplateMaps} templateMaps * @param {MapBackend} mapBackend * @param metadataBackend - * @param {OverviewsMetadataApi} overviewsMetadataApi * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigOverviewsAdapter} overviewsAdapter + * @param {TurboCartocssAdapter} turboCartoCssAdapter + * @param {AnalysisBackend} analysisBackend * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, - overviewsAdapter, turboCartoCssAdapter) { + overviewsAdapter, turboCartoCssAdapter, analysisBackend) { BaseController.call(this, authApi, pgConnection); @@ -47,7 +49,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.layergroupAffectedTables = layergroupAffectedTables; this.turboCartoCssAdapter = turboCartoCssAdapter; - this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(); + this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend); this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); this.overviewsAdapter = overviewsAdapter; } diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis_mapconfig_adapter.js index 1fcab837..7f45fd89 100644 --- a/lib/cartodb/models/analysis_mapconfig_adapter.js +++ b/lib/cartodb/models/analysis_mapconfig_adapter.js @@ -5,7 +5,8 @@ var camshaft = require('camshaft'); var dot = require('dot'); dot.templateSettings.strip = false; -function AnalysisMapConfigAdapter() { +function AnalysisMapConfigAdapter(analysisBackend) { + this.analysisBackend = analysisBackend; } module.exports = AnalysisMapConfigAdapter; @@ -21,12 +22,13 @@ function skipColumns(columnNames) { } var layerQueryTemplate = dot.template([ - 'SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, {{=it._columns}}', + 'SELECT {{=it._columns}}', 'FROM ({{=it._query}}) _cdb_analysis_query' ].join('\n')); function layerQuery(query, columnNames) { - return layerQueryTemplate({ _query: query, _columns: skipColumns(columnNames).join(', ') }); + var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(columnNames)); + return layerQueryTemplate({ _query: query, _columns: _columns.join(', ') }); } function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { @@ -66,6 +68,7 @@ function getFilter(dataview, params) { AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, filters, callback) { // jshint maxcomplexity:7 + var self = this; filters = filters || {}; var dataviewsFilters = filters.dataviews || {}; debug(dataviewsFilters); @@ -95,7 +98,7 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r } function createAnalysis(analysisDefinition, done) { - camshaft.create(analysisConfiguration, analysisDefinition, done); + self.analysisBackend.create(analysisConfiguration, analysisDefinition, done); } var analysesQueue = queue(requestMapConfig.analyses.length); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index e04f835d..e727f4b2 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -27,6 +27,8 @@ var NamedMapProviderCache = require('./cache/named_map_provider_cache'); var PgQueryRunner = require('./backends/pg_query_runner'); var PgConnection = require('./backends/pg_connection'); +var AnalysisBackend = require('./backends/analysis'); + var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); @@ -140,6 +142,7 @@ module.exports = function(serverOptions) { var tileBackend = new windshaft.backend.Tile(rendererCache); var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + var analysisBackend = new AnalysisBackend(serverOptions.analysis); var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; @@ -181,7 +184,8 @@ module.exports = function(serverOptions) { new windshaft.backend.Widget(), surrogateKeysCache, userLimitsApi, - layergroupAffectedTablesCache + layergroupAffectedTablesCache, + analysisBackend ).register(app); new controller.Map( @@ -194,7 +198,8 @@ module.exports = function(serverOptions) { userLimitsApi, layergroupAffectedTablesCache, overviewsAdapter, - turboCartocssAdapter + turboCartocssAdapter, + analysisBackend ).register(app); new controller.NamedMaps( diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 00011356..f129807c 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -31,6 +31,12 @@ if (global.environment.statsd) { } } +var analysisConfig = _.defaults(global.environment.analysis || {}, { + batch: { + endpoint: 'http://127.0.0.1:8080/api/v1/sql/job' + } +}); + module.exports = { bind: { port: global.environment.port, @@ -81,6 +87,12 @@ module.exports = { torque: rendererConfig.torque, http: rendererConfig.http }, + + analysis: { + batch: { + endpoint: analysisConfig.batch.endpoint + } + }, // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161 redis: _.extend(global.environment.redis, {unwatchOnRelease: false}), enable_cors: global.environment.enable_cors, diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index ffcc5fa3..d56a2aa9 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -161,7 +161,7 @@ describe('analysis-layers', function() { }); }); - it('should fail for non-authenticated requests that has a node other than "source"', function(done) { + it.skip('should fail for non-authenticated requests that has a node other than "source"', function(done) { var useCase = useCases[1]; // No API key here diff --git a/test/support/test-client.js b/test/support/test-client.js index b83d0589..7372ac82 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -12,8 +12,30 @@ var helper = require('./test_helper'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); +function createServiceStub(result) { + return function(__, callback) { + return callback(null, result); + }; +} +function AnalysisDatabaseServiceStub(/*dbParams, batchParams*/) { +// this.dbParams = dbParams; +// this.batchParams = batchParams; +} +AnalysisDatabaseServiceStub.prototype = { + run: createServiceStub({}), + getSchema: createServiceStub([]), + getColumnNames: createServiceStub([]), + getLastUpdatedTimeFromAffectedTables: createServiceStub([]), + setUpdatedAtForSources: createServiceStub([]), + registerAnalysisInCatalog: createServiceStub([]), + queueAnalysisOperations: createServiceStub([]), + trackAnalysis: createServiceStub([]), + enqueue: createServiceStub({}) +}; +serverOptions.analysis.batch.databaseService = AnalysisDatabaseServiceStub; + +var server = new CartodbWindshaft(serverOptions); function TestClient(mapConfig, apiKey) { this.mapConfig = mapConfig; From da602eeda06381ffc0735eafe68f83195b7ef4a4 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 14 Apr 2016 17:25:08 +0200 Subject: [PATCH 70/80] Use inline execution in camshaft instead of a database service stub --- lib/cartodb/backends/analysis.js | 13 +++++------ test/acceptance/analysis/analysis-layers.js | 2 +- test/support/test-client.js | 24 +-------------------- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js index 2133e772..dc06a8ac 100644 --- a/lib/cartodb/backends/analysis.js +++ b/lib/cartodb/backends/analysis.js @@ -2,15 +2,16 @@ var camshaft = require('camshaft'); function AnalysisBackend(options) { var batchConfig = options.batch || {}; - this.batchEndpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job'; - - var databaseService = batchConfig.databaseService || null; - this.analysisFactory = (databaseService === null) ? camshaft : new camshaft(databaseService); + batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job'; + batchConfig.inlineExecution = batchConfig.inlineExecution || false; + this.batchConfig = batchConfig; } module.exports = AnalysisBackend; AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) { - analysisConfiguration.batch.endpoint = this.batchEndpoint; - this.analysisFactory.create(analysisConfiguration, analysisDefinition, callback); + analysisConfiguration.batch.endpoint = this.batchConfig.endpoint; + analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution; + + camshaft.create(analysisConfiguration, analysisDefinition, callback); }; diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index d56a2aa9..ffcc5fa3 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -161,7 +161,7 @@ describe('analysis-layers', function() { }); }); - it.skip('should fail for non-authenticated requests that has a node other than "source"', function(done) { + it('should fail for non-authenticated requests that has a node other than "source"', function(done) { var useCase = useCases[1]; // No API key here diff --git a/test/support/test-client.js b/test/support/test-client.js index 7372ac82..1da653a7 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -12,29 +12,7 @@ var helper = require('./test_helper'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); - -function createServiceStub(result) { - return function(__, callback) { - return callback(null, result); - }; -} -function AnalysisDatabaseServiceStub(/*dbParams, batchParams*/) { -// this.dbParams = dbParams; -// this.batchParams = batchParams; -} -AnalysisDatabaseServiceStub.prototype = { - run: createServiceStub({}), - getSchema: createServiceStub([]), - getColumnNames: createServiceStub([]), - getLastUpdatedTimeFromAffectedTables: createServiceStub([]), - setUpdatedAtForSources: createServiceStub([]), - registerAnalysisInCatalog: createServiceStub([]), - queueAnalysisOperations: createServiceStub([]), - trackAnalysis: createServiceStub([]), - enqueue: createServiceStub({}) -}; -serverOptions.analysis.batch.databaseService = AnalysisDatabaseServiceStub; - +serverOptions.analysis.batch.inlineExecution = true; var server = new CartodbWindshaft(serverOptions); function TestClient(mapConfig, apiKey) { From 68b19c65fee1132d5787388f2aff6e74e9d13654 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Apr 2016 14:40:34 +0200 Subject: [PATCH 71/80] newline at end of file --- lib/cartodb/backends/analysis-status.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/analysis-status.js b/lib/cartodb/backends/analysis-status.js index 8ba7ae9d..211157d8 100644 --- a/lib/cartodb/backends/analysis-status.js +++ b/lib/cartodb/backends/analysis-status.js @@ -46,4 +46,4 @@ function dbParamsFromReqParams(params) { dbParams.dbname = params.dbname; } return dbParams; -} \ No newline at end of file +} From 5baad96924f34bdf7a0029d5633e097ddde6eeda Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Apr 2016 14:43:29 +0200 Subject: [PATCH 72/80] remove commented out code --- test/acceptance/analysis/named-maps.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js index 17447e5c..f3e2aca0 100644 --- a/test/acceptance/analysis/named-maps.js +++ b/test/acceptance/analysis/named-maps.js @@ -1,8 +1,6 @@ var assert = require('../../support/assert'); var step = require('step'); -//var mapnik = require('windshaft').mapnik; - var helper = require('../../support/test_helper'); var CartodbWindshaft = require('../../../lib/cartodb/server'); @@ -155,7 +153,7 @@ describe('named-maps analysis', function() { ); }); - it('should be able to retrieve widgets from all URLs', function(done) { + it('should be able to retrieve images from analysis', function(done) { assert.response( server, { @@ -177,8 +175,6 @@ describe('named-maps analysis', function() { return done(err); } -// var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); -// assert.ok(image); var fixturePath = './test/fixtures/analysis/named-map-buffer.png'; assert.imageBufferIsSimilarToFile(res.body, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) { assert.ok(!err, err); From 263b3e368294d58d4ad9e696f3f237c224e931ff Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Apr 2016 14:47:35 +0200 Subject: [PATCH 73/80] Rename file --- lib/cartodb/cache/named_map_provider_cache.js | 2 +- lib/cartodb/controllers/map.js | 2 +- ...lysis_mapconfig_adapter.js => analysis-mapconfig-adapter.js} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename lib/cartodb/models/{analysis_mapconfig_adapter.js => analysis-mapconfig-adapter.js} (100%) diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index 80195e41..301f21ce 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -2,7 +2,7 @@ var _ = require('underscore'); var dot = require('dot'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); -var AnalysisMapConfigAdapter = require('../models/analysis_mapconfig_adapter'); +var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter'); var templateName = require('../backends/template_maps').templateName; var queue = require('queue-async'); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 32c5203a..3156e723 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -16,7 +16,7 @@ var Datasource = windshaft.model.Datasource; var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); -var AnalysisMapConfigAdapter = require('../models/analysis_mapconfig_adapter'); +var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); diff --git a/lib/cartodb/models/analysis_mapconfig_adapter.js b/lib/cartodb/models/analysis-mapconfig-adapter.js similarity index 100% rename from lib/cartodb/models/analysis_mapconfig_adapter.js rename to lib/cartodb/models/analysis-mapconfig-adapter.js From ab55b083b44b50ab7967cd158390b6e61f093cd6 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Apr 2016 14:48:14 +0200 Subject: [PATCH 74/80] Style --- lib/cartodb/controllers/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 3156e723..ff6a0f1c 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -286,7 +286,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, [],layergroup, this); + self.afterLayergroupCreate(req, res, mapConfig, [], layergroup, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { From f932862ce4d1338fb4d88381d4c6e61165670f8a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Apr 2016 15:13:00 +0200 Subject: [PATCH 75/80] Improve configuration for batch queries --- config/environments/development.js.example | 14 ++++++++++++++ config/environments/production.js.example | 14 ++++++++++++++ config/environments/staging.js.example | 14 ++++++++++++++ config/environments/test.js.example | 14 ++++++++++++++ lib/cartodb/backends/dataview.js | 2 -- lib/cartodb/controllers/map.js | 2 -- lib/cartodb/models/mapconfig/named_map_provider.js | 2 -- lib/cartodb/server_options.js | 4 +++- 8 files changed, 59 insertions(+), 7 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 8a1cc7ad..d458976b 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -191,6 +191,20 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: false, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 73c68f1d..bfed10ff 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -185,6 +185,20 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: false, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/home/ubuntu/tile_assets/' diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index f81b88f7..6886d028 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -185,6 +185,20 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: false, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/home/ubuntu/tile_assets/' diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 0400be6e..758bfcb4 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -186,6 +186,20 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: true, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/tmp/cdb-tiler-test/millstone' diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index f07ed6d1..510709c4 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -56,8 +56,6 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param pass: params.dbpassword }, batch: { - // TODO load this from configuration - endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', username: user, apiKey: params.api_key } diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ff6a0f1c..aed91641 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -150,8 +150,6 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { pass: req.params.dbpassword }, batch: { - // TODO load this from configuration - endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', username: req.context.user, apiKey: req.params.api_key } diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index b4646f7b..b1eaba8b 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -105,8 +105,6 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { pass: rendererParams.dbpassword }, batch: { - // TODO load this from configuration - endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', username: self.owner, apiKey: apiKey } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index f129807c..a75bba58 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -33,7 +33,8 @@ if (global.environment.statsd) { var analysisConfig = _.defaults(global.environment.analysis || {}, { batch: { - endpoint: 'http://127.0.0.1:8080/api/v1/sql/job' + inlineExecution: false, + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' } }); @@ -90,6 +91,7 @@ module.exports = { analysis: { batch: { + inlineExecution: analysisConfig.batch.inlineExecution, endpoint: analysisConfig.batch.endpoint } }, From d910f5ef3e5a82b1c3caff8c38a1a138b2f18a8d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 20 Apr 2016 15:33:04 +0200 Subject: [PATCH 76/80] Update camshaft to use 0.4.0 fixed version --- npm-shrinkwrap.json | 18 +++++++++--------- package.json | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index dd3845d0..f8677e7c 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -34,7 +34,7 @@ }, "statuses": { "version": "1.2.1", - "from": "statuses@>=1.0.0 <2.0.0", + "from": "statuses@>=1.2.1 <1.3.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz" } } @@ -90,7 +90,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.2 <2.2.0", + "from": "mime-types@>=2.1.10 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -105,9 +105,9 @@ } }, "camshaft": { - "version": "0.3.0", - "from": "https://github.com/CartoDB/camshaft/tarball/master", - "resolved": "https://github.com/CartoDB/camshaft/tarball/master", + "version": "0.4.0", + "from": "camshaft@0.4.0", + "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.4.0.tgz", "dependencies": { "async": { "version": "1.5.2", @@ -664,7 +664,7 @@ "dependencies": { "unpipe": { "version": "1.0.0", - "from": "unpipe@>=1.0.0 <1.1.0", + "from": "unpipe@1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" } } @@ -779,7 +779,7 @@ }, "type-is": { "version": "1.6.12", - "from": "type-is@>=1.6.6 <1.7.0", + "from": "type-is@>=1.6.10 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { @@ -1112,7 +1112,7 @@ }, "isstream": { "version": "0.1.2", - "from": "isstream@>=0.1.2 <0.2.0", + "from": "isstream@>=0.1.1 <0.2.0", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" }, "har-validator": { @@ -1455,7 +1455,7 @@ }, "forever-agent": { "version": "0.6.1", - "from": "forever-agent@>=0.6.0 <0.7.0", + "from": "forever-agent@>=0.6.1 <0.7.0", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" }, "form-data": { diff --git a/package.json b/package.json index 8f9e3b00..a3bc9ae9 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "request": "~2.62.0", "cartodb-redis": "~0.13.0", "cartodb-psql": "~0.6.1", - "camshaft": "https://github.com/CartoDB/camshaft/tarball/master", + "camshaft": "0.4.0", "fastly-purge": "~1.0.1", "redis-mpool": "~0.4.0", "lru-cache": "2.6.5", From d70af7c9c1573471650181db395f950a496d427d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 20 Apr 2016 15:33:17 +0200 Subject: [PATCH 77/80] Fix tests with typo in s/radio/radius/ --- .../acceptance/analysis/analysis-layers-use-cases.js | 12 ++++++------ test/acceptance/analysis/analysis-layers.js | 2 +- test/acceptance/analysis/named-maps.js | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 6084b88c..e60c8e79 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -121,7 +121,7 @@ describe('analysis-layers use cases', function() { "query": "select * from analysis_banks" } }, - "radio": 250 + "radius": 250 } }), cartocss: cartocss('black', 0.5) @@ -162,7 +162,7 @@ describe('analysis-layers use cases', function() { "query": "select * from analysis_banks" } }, - "radio": 250 + "radius": 250 } } } @@ -197,7 +197,7 @@ describe('analysis-layers use cases', function() { "query": "select * from analysis_banks" } }, - "radio": 250 + "radius": 250 } } } @@ -239,7 +239,7 @@ describe('analysis-layers use cases', function() { "query": "select * from analysis_banks" } }, - "radio": 300 + "radius": 300 } }), cartocss: cartocss('magenta', 0.5) @@ -266,7 +266,7 @@ describe('analysis-layers use cases', function() { "query": "select * from analysis_banks" } }, - "radio": 300 + "radius": 300 } } } @@ -340,7 +340,7 @@ describe('analysis-layers use cases', function() { query: "select * from analysis_banks" } }, - "radio": 250 + "radius": 250 }, dataviews: { bank_category: { diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index ffcc5fa3..f2a3569f 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -115,7 +115,7 @@ describe('analysis-layers', function() { "query": "select * from populated_places_simple_reduced" } }, - "radio": 50000 + "radius": 50000 } } ] diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js index f3e2aca0..197c281f 100644 --- a/test/acceptance/analysis/named-maps.js +++ b/test/acceptance/analysis/named-maps.js @@ -52,7 +52,7 @@ describe('named-maps analysis', function() { "query": "select * from populated_places_simple_reduced" } }, - "radio": 50000 + "radius": 50000 } } ] From e04a9a25797785dc7b1cca2932a9be59b3ef67e0 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 20 Apr 2016 15:40:14 +0200 Subject: [PATCH 78/80] Append dataviews filters after checking if mapconfig must be adapted --- lib/cartodb/models/analysis-mapconfig-adapter.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/analysis-mapconfig-adapter.js b/lib/cartodb/models/analysis-mapconfig-adapter.js index 7f45fd89..3c5761c6 100644 --- a/lib/cartodb/models/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/analysis-mapconfig-adapter.js @@ -70,6 +70,11 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r // jshint maxcomplexity:7 var self = this; filters = filters || {}; + + if (!shouldAdaptLayers(requestMapConfig)) { + return callback(null, requestMapConfig); + } + var dataviewsFilters = filters.dataviews || {}; debug(dataviewsFilters); var dataviews = requestMapConfig.dataviews || {}; @@ -93,10 +98,6 @@ AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, r requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); - if (!shouldAdaptLayers(requestMapConfig)) { - return callback(null, requestMapConfig); - } - function createAnalysis(analysisDefinition, done) { self.analysisBackend.create(analysisConfiguration, analysisDefinition, done); } From 41ead9664bc713cdb7b251c31703b26ef9a7ec92 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 20 Apr 2016 16:34:48 +0200 Subject: [PATCH 79/80] Upgrade camshaft to 0.5.0 --- npm-shrinkwrap.json | 12 ++++++------ package.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index f8677e7c..25c12d5b 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -80,7 +80,7 @@ }, "type-is": { "version": "1.6.12", - "from": "type-is@>=1.6.10 <1.7.0", + "from": "type-is@>=1.6.6 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { @@ -90,7 +90,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.2 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -105,9 +105,9 @@ } }, "camshaft": { - "version": "0.4.0", - "from": "camshaft@0.4.0", - "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.4.0.tgz", + "version": "0.5.0", + "from": "camshaft@0.5.0", + "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.5.0.tgz", "dependencies": { "async": { "version": "1.5.2", @@ -779,7 +779,7 @@ }, "type-is": { "version": "1.6.12", - "from": "type-is@>=1.6.10 <1.7.0", + "from": "type-is@>=1.6.6 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.12.tgz", "dependencies": { "media-typer": { diff --git a/package.json b/package.json index a3bc9ae9..3cd3ad82 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "request": "~2.62.0", "cartodb-redis": "~0.13.0", "cartodb-psql": "~0.6.1", - "camshaft": "0.4.0", + "camshaft": "0.5.0", "fastly-purge": "~1.0.1", "redis-mpool": "~0.4.0", "lru-cache": "2.6.5", From a35403cd91fee2b87e7baa9e8a75167dd64a230a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 20 Apr 2016 16:36:29 +0200 Subject: [PATCH 80/80] Add option to modify host header template for camshaft batch client --- config/environments/development.js.example | 4 +++- config/environments/production.js.example | 4 +++- config/environments/staging.js.example | 4 +++- config/environments/test.js.example | 4 +++- lib/cartodb/backends/analysis.js | 2 ++ lib/cartodb/server_options.js | 6 ++++-- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index d458976b..480f15b8 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -202,7 +202,9 @@ var config = { // - running an standalone server without any dependency on external services inlineExecution: false, // where the SQL API is running, it will use a custom Host header to specify the username. - endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' } } ,millstone: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index bfed10ff..2a457768 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -196,7 +196,9 @@ var config = { // - running an standalone server without any dependency on external services inlineExecution: false, // where the SQL API is running, it will use a custom Host header to specify the username. - endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' } } ,millstone: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 6886d028..72195fef 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -196,7 +196,9 @@ var config = { // - running an standalone server without any dependency on external services inlineExecution: false, // where the SQL API is running, it will use a custom Host header to specify the username. - endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' } } ,millstone: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 758bfcb4..00c19628 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -197,7 +197,9 @@ var config = { // - running an standalone server without any dependency on external services inlineExecution: true, // where the SQL API is running, it will use a custom Host header to specify the username. - endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' } } ,millstone: { diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js index dc06a8ac..5855da46 100644 --- a/lib/cartodb/backends/analysis.js +++ b/lib/cartodb/backends/analysis.js @@ -4,6 +4,7 @@ function AnalysisBackend(options) { var batchConfig = options.batch || {}; batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job'; batchConfig.inlineExecution = batchConfig.inlineExecution || false; + batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan'; this.batchConfig = batchConfig; } @@ -12,6 +13,7 @@ module.exports = AnalysisBackend; AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) { analysisConfiguration.batch.endpoint = this.batchConfig.endpoint; analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution; + analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate; camshaft.create(analysisConfiguration, analysisDefinition, callback); }; diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index a75bba58..22039b8c 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -34,7 +34,8 @@ if (global.environment.statsd) { var analysisConfig = _.defaults(global.environment.analysis || {}, { batch: { inlineExecution: false, - endpoint: 'http://127.0.0.1:8080/api/v2/sql/job' + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + hostHeaderTemplate: '{{=it.username}}.localhost.lan' } }); @@ -92,7 +93,8 @@ module.exports = { analysis: { batch: { inlineExecution: analysisConfig.batch.inlineExecution, - endpoint: analysisConfig.batch.endpoint + endpoint: analysisConfig.batch.endpoint, + hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate } }, // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161