From 8012d76b68eafbd40d0aadb3a1b1ac79d3077fbb Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Fri, 22 Dec 2017 15:58:40 +0100 Subject: [PATCH 1/8] Timeseries: Avoid collisions with columns named 'timestamp' --- lib/cartodb/models/dataview/histograms/date-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 8fd6ff30..4c720b3b 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -56,7 +56,7 @@ __wd_buckets AS ${ctx.query} ) __source, __wd_tz ${condition_str} - GROUP BY timestamp, __wd_tz.name + GROUP BY 1, __wd_tz.name ),`; } From 031e1253ca11aa03282f612f2f2d9a4736d5b5ef Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Fri, 22 Dec 2017 17:12:57 +0100 Subject: [PATCH 2/8] Numeric histograms: Avoid conflicts with 'bin' named columns --- lib/cartodb/models/dataview/histograms/numeric-histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index a2d5662d..d3eedad9 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -28,7 +28,7 @@ const binsQueryTpl = ctx => ` ) AS quartile FROM __cdb_filtered_source) _cdb_quartiles WHERE quartile = 1 or quartile = 3 - GROUP BY quartile + GROUP BY 1 ) __cdb_iqr ), __cdb_bins AS ( @@ -137,8 +137,8 @@ FROM ( ${ctx.query} ) __cdb_filtered_source_query${extra_tables} -GROUP BY bin${extra_groupby} -ORDER BY bin;`; +GROUP BY 10${extra_groupby} +ORDER BY 10;`; } _hasOverridenBins (override) { From ef3917fa6f729a353da831114221c6146c8ecdea Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 22 Dec 2017 17:18:42 +0100 Subject: [PATCH 3/8] Update vector tiles documentation This update reflects the fact that CartoCSS is now optional for vector tiles. --- docs/anonymous_maps.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/anonymous_maps.md b/docs/anonymous_maps.md index 1dc2f5f3..9ad03487 100644 --- a/docs/anonymous_maps.md +++ b/docs/anonymous_maps.md @@ -1,6 +1,7 @@ # Anonymous Maps -Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec) +Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec). +Alternatively, you can get the data for the map (geometry and attributes for each layer) using vector tiles (in which case CartoCSS is not required). ## Instantiate @@ -72,19 +73,19 @@ curl 'https://{username}.carto.com/api/v1/map' -H 'Content-Type: application/jso ## Map Tile Rendering -Map tiles create the graphical representation of your map in a web browser. The performance rendering of map tiles is dependent on the type of geospatial data model (raster or vector) that you are using. +Map tiles are used to create the graphic representation of your map in a web browser. Tiles can be requested either as pre-rendered *raster* tiles (images) or as *vector* map data to be rendered by the client (browser). -- **Raster**: Generates map tiles based on a grid of pixels to represent your data. Each cell is a fixed size and contains values for particular map features. On the server-side, each request queries a dataset to retrieve data for each map tile. The grid size of map tiles can often lead to graphic quality issues. +- **Raster**: If a tile is requested as a raster image format, like PNG, the map will be rendered on the server, using the CartoCSS styles defined in the layers of the map. It is necessary that all the layers of a map define CartoCSS styles in order to obtain raster tiles. Raster tiles are made up of 256x256 pixels; to avoid graphic quality issues tiles should be used unscaled to represent the zoom level (Z) for which they are requested. In order to render tiles, data will be retrieved from the database (in vector format) on the server-side. -- **Vector**: Generates map tiles based on pre-defined coordinates to represent your data, similar to how basemap image tiles are rendered. On the client-side, map tiles represent real-world geometries of a map. Depending on the coordinates, vertices are used to connect the data and display points, lines, or polygons for the map tiles. +- **Vector**: Tiles can also be requested as MVT (Mapbox Vector Tiles). In this case, only the geospatial vector data, without any styling, is returned. These tiles should be processed in the client-side to render the map. In this case layers do not need to define CartoCSS, as any rendering and styling will be performed on the client side. The vector data of a tile represents real-world geometries by defining the vertices of points, lines or polygons in a tile-specific coordinate system. ## Retrieve resources from the layergroup When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection. -### Mapnik tiles +### Raster tiles -These raster tiles retrieve just the Mapnik layers. See [individual layers](#individual-layers) for details about how to retrieve other layers. +These raster tiles are PNG images that represent only the Mapnik layers of a map. See [individual layers](#individual-layers) for details about how to retrieve other layers. ```bash https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png @@ -92,9 +93,9 @@ https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png ### Mapbox Vector Tiles (MVT) -[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that store geographic vector data on the client-side. Browser performance is fast since you can pan and zoom without having to query the server. +[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that transfer geographic vector data to the client-side. Browser performance is fast since you can pan and zoom without having to query the server. -CARTO uses a Web Graphics Library (WebGL) to process MVT files. This is useful since WebGL's are compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files. +CARTO uses Web Graphics Library (WebGL) to process MVT files on the browser. This is useful since WebGL is compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files. The following examples describe how to fetch MVT tiles with a cURL request. @@ -245,7 +246,7 @@ center: [30, 0] map.setStyle({ "version": 7, "glyphs": "...", - "constants": {...}, + "constants": {...}, "sources": { "cartodb": { "type": "vector", From ca5f280cb3704e006b572cbc1a4f9b12e05d45fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 26 Dec 2017 11:03:12 +0100 Subject: [PATCH 4/8] updating NEWS.md --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 1b8f66d8..968646b7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ Released YYYY-MM-DD Announcements: - Validate aggregation input params + - Fix column names collisions in histograms [#828](https://github.com/CartoDB/Windshaft-cartodb/pull/828) ## 4.5.0 From 89333185a92209ab1f9d6b937040e8b687b45559 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 28 Dec 2017 16:37:17 +0000 Subject: [PATCH 5/8] Make tests to fail if they got an uncaught exception --- test/support/test_helper.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/support/test_helper.js b/test/support/test_helper.js index 49c6c1ee..96a109ff 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -75,6 +75,22 @@ function checkSurrogateKey(res, expectedKey) { assert.deepEqual(keys, expectedKeys); } +var uncaughtExceptions = []; +process.on('uncaughtException', function(err) { + uncaughtExceptions.push(err); +}); +beforeEach(function() { + uncaughtExceptions = []; +}); +//global afterEach to capture uncaught exceptions +afterEach(function() { + assert.equal( + uncaughtExceptions.length, + 0, + 'uncaughtException:\n\n' + uncaughtExceptions.map(err => err.stack).join('\n\n')); +}); + + var redisClient; beforeEach(function() { From d7f4d39aa2631e407585dd6f5b48138d8343026e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 28 Dec 2017 18:09:41 +0000 Subject: [PATCH 6/8] Don't not override the full client but only the provided methods --- test/unit/cartodb/ported/tile_stats.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/cartodb/ported/tile_stats.test.js b/test/unit/cartodb/ported/tile_stats.test.js index 3fbf93ec..a2d30cd0 100644 --- a/test/unit/cartodb/ported/tile_stats.test.js +++ b/test/unit/cartodb/ported/tile_stats.test.js @@ -11,7 +11,7 @@ describe('tile stats', function() { }); afterEach(function() { - global.statsClient = this.statsClient; + global.statsClient = this.statsClient; }); it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() { @@ -84,7 +84,7 @@ describe('tile stats', function() { }); function mockStatsClientGetInstance(instance) { - global.statsClient = instance; + global.statsClient = Object.assign(global.statsClient, instance); } }); From e43ccf4f1274ec4dd384cdfac77f647e7dfb78d3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 28 Dec 2017 18:19:52 +0000 Subject: [PATCH 7/8] Going red: individual layer id filters fail Depends on https://github.com/CartoDB/Windshaft/pull/584. --- test/acceptance/layers-filters.js | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 test/acceptance/layers-filters.js diff --git a/test/acceptance/layers-filters.js b/test/acceptance/layers-filters.js new file mode 100644 index 00000000..fa42c900 --- /dev/null +++ b/test/acceptance/layers-filters.js @@ -0,0 +1,54 @@ +require('../support/test_helper'); +var TestClient = require('../support/test-client'); + +describe('layers filters', function() { + const type = 'mapnik'; + const sql = 'select * from populated_places_simple_reduced'; + const cartocss = `#points { + marker-fill-opacity: 1.0; + marker-line-color: #FFF; + marker-line-width: 0.5; + marker-line-opacity: 1.0; + marker-placement: point; + marker-type: ellipse; + marker-width: 8; + marker-fill: red; + marker-allow-overlap: true; + }`; + const cartocss_version = '3.0.12'; + const options = { + sql, + cartocss, + cartocss_version + }; + + const mapConfig = { + version: '1.6.0', + layers: [ + { + type, + id: 'layerA', + options + }, + { + type, + id: 'layerB', + options + } + ] + }; + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } + }); + + ['layerA', 'layerB'].forEach(layer => { + it(`should work for individual layer ids: ${layer}`, function (done) { + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, { layers: layer }, done); + }); + }); + +}); \ No newline at end of file From a75beefe6e95f54aa9096946a2609dba03591346 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 28 Dec 2017 18:34:53 +0000 Subject: [PATCH 8/8] Upgrades windshaft to 4.1.1 --- NEWS.md | 1 + package.json | 2 +- yarn.lock | 16 ++++++++-------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/NEWS.md b/NEWS.md index 968646b7..24a86655 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,7 @@ Released YYYY-MM-DD Announcements: + - Upgrades windshaft to [4.1.1](https://github.com/CartoDB/windshaft/releases/tag/4.1.1). - Validate aggregation input params - Fix column names collisions in histograms [#828](https://github.com/CartoDB/Windshaft-cartodb/pull/828) diff --git a/package.json b/package.json index dd210fc0..f7386669 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.1.0", + "windshaft": "4.1.1", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 78e663f0..26e59383 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -226,7 +226,7 @@ camshaft@0.60.0: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -252,7 +252,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -2223,7 +2223,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2231,7 +2231,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2392,9 +2392,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.1.0.tgz#dc17c8369570c305171d1ab5ca130369bba04d58" +windshaft@4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.1.1.tgz#cdae06202e841b8e75915de5155b8a279b4547be" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2