Merge branch 'master' into cartonik

2019-04-25 14:14:16 +02:00 · 2019-04-25 14:14:16 +02:00 · f93ea0e95c
commit f93ea0e95c
parent da613df71c b90e15207f
11 changed files with 242 additions and 55 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,8 +1,12 @@
-jobs:
-  include:
-    - sudo: required
-      services:
-        - docker
-      language: generic
-      before_install: docker pull carto/nodejs-xenial-pg101:latest
-      script: npm run docker-test -- 10.15.1 # Node.js version
+language: generic
+sudo: required
+env:
+  matrix:
+    - NODE_VERSION=10.15.1
+      DOCKER_IMAGE=carto/nodejs-xenial-pg101:latest
+    - NODE_VERSION=10.15.1
+      DOCKER_IMAGE=carto/nodejs-xenial-pg1121:latest
+services:
+  - docker
+before_install: docker pull ${DOCKER_IMAGE}
+script: npm run docker-test -- ${DOCKER_IMAGE} ${NODE_VERSION}
--- a/NEWS.md
+++ b/NEWS.md
@ -13,12 +13,13 @@ Announcements:
 - Fix boolean aggregation layer option not working when numbers of rows are above the threshold (#1082)
 - Update deps:
  - camshat@0.64.0
-  - windshaft@5.1.0:
+  - windshaft@5.1.1:
    - Upgrade `tilelive-mapnik` to version `0.6.18-cdb20`
    - Upgrade `@carto/tilelive-bridge` to version `2.5.1-cdb12`
    - Upgrade `grainstore` to version `2.0.0`
    - Upgrade `torque.js` to version `3.1.0`
    - Upgrade `canvas` to version `2.4.1`
+    - Update @carto/mapnik to [`3.6.2-carto.13`](https://github.com/CartoDB/node-mapnik/blob/v3.6.2-carto.13/CHANGELOG.carto.md#362-carto13).


 ## 7.0.0
--- a/docker-test.sh
+++ b/docker-test.sh
@ -1,4 +1,4 @@
 #!/bin/bash

-docker run -e "NODEJS_VERSION=${1}" -v `pwd`:/srv carto/nodejs-xenial-pg101:latest bash run_tests_docker.sh && \
+docker run -e "NODEJS_VERSION=${2}" -v `pwd`:/srv ${1} bash run_tests_docker.sh && \
    docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v
--- a/docker/Dockerfile-nodejs-xenial-pg1121:latest
+++ b/docker/Dockerfile-nodejs-xenial-pg1121:latest
@ -0,0 +1,85 @@
+FROM ubuntu:xenial
+
+# Use UTF8 to avoid encoding problems with pgsql
+ENV LANG C.UTF-8
+ENV NPROCS 1
+ENV JOBS 1
+ENV CXX g++-4.9
+ENV PGUSER postgres
+
+# Add external repos
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        curl \
+        software-properties-common \
+        locales \
+    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+    && add-apt-repository -y ppa:cartodb/postgresql-11 \
+    && add-apt-repository -y ppa:cartodb/redis-next \
+    && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash \
+    && . ~/.nvm/nvm.sh \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        g++-4.9 \
+        gcc-4.9 \
+        git \
+        libcairo2-dev \
+        libgdal-dev=2.3.2+dfsg-2build2~carto1 \
+        libgdal20=2.3.2+dfsg-2build2~carto1  \
+        libgeos-dev=3.7.1~carto1 \
+        libgif-dev \
+        libjpeg8-dev \
+        libjson-c-dev \
+        libpango1.0-dev \
+        libpixman-1-dev \
+        libproj-dev \
+        libprotobuf-c-dev \
+        libxml2-dev \
+        gdal-bin=2.3.2+dfsg-2build2~carto1 \
+        make \
+        nodejs \
+        protobuf-c-compiler \
+        pkg-config \
+        wget \
+        zip \
+        libopenscenegraph100v5 \
+        libsfcgal1 \
+        liblwgeom-2.5.0=2.5.1.4+carto-1 \
+        postgresql-11 \
+        postgresql-11-plproxy \
+        postgis=2.5.1.4+carto-1 \
+        postgresql-11-postgis-2.5=2.5.1.4+carto-1 \
+        postgresql-11-postgis-2.5-scripts=2.5.1.4+carto-1 \
+        postgresql-client-11 \
+        postgresql-client-common \
+        postgresql-common \
+        postgresql-contrib \
+        postgresql-plpython-11 \
+        postgresql-server-dev-11 \
+        redis=5:4.0.9-1carto1~xenial1 \
+    && apt-get purge -y wget protobuf-c-compiler \
+    && apt-get autoremove -y
+
+# Configure PostgreSQL
+RUN set -ex \
+    && echo "listen_addresses='*'" >> /etc/postgresql/11/main/postgresql.conf \
+    && echo "local     all       all                     trust" >  /etc/postgresql/11/main/pg_hba.conf \
+    && echo "host      all       all       0.0.0.0/0     trust" >> /etc/postgresql/11/main/pg_hba.conf \
+    && echo "host      all       all       ::1/128       trust" >> /etc/postgresql/11/main/pg_hba.conf \
+    && /etc/init.d/postgresql start \
+    && createdb template_postgis \
+    && createuser publicuser \
+    && psql -c "CREATE EXTENSION postgis" template_postgis \
+    && /etc/init.d/postgresql stop
+
+WORKDIR /srv
+EXPOSE 5858
+
+COPY ./scripts/nodejs-install.sh /src/nodejs-install.sh
+RUN chmod 777 /src/nodejs-install.sh
+CMD /src/nodejs-install.sh
--- a/docs/guides/02-general-concepts.md
+++ b/docs/guides/02-general-concepts.md
@ -37,10 +37,10 @@ All the endpoints, which might be accessed using a web browser, add CORS headers

 **Note:** By default, CARTO uses vector graphics for map rendering. Please [contact us](mailto:support@carto.com) if you need raster rendering enabled as part of your requirements.

- ### Mapbox Vector Tiles (MVT)
+### Mapbox Vector Tiles (MVT)

 [Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that store geographic vector data on the client-side. Browser performance is fast since you can pan and zoom without having to query the server.

 CARTO uses a Web Graphics Library (WebGL) to process MVT files. This is useful since WebGL's are compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles.

- **Tip:** You can process MVT files with the [`ST_AsMVT` PostGIS function](https://postgis.net/docs/manual-dev/ST_AsMVT.html) with the [Maps API Windshaft renderer](https://github.com/CartoDB/Windshaft/blob/1000x/lib/windshaft/renderers/pg_mvt/renderer.js).
+ **Tip:** You can process MVT files with the [`ST_AsMVT` PostGIS function](https://postgis.net/docs/manual-dev/ST_AsMVT.html) with the [Maps API Windshaft renderer](https://github.com/CartoDB/Windshaft/blob/1000x/lib/windshaft/renderers/pg_mvt/renderer.js).
--- a/package-lock.json
+++ b/package-lock.json
@ -26,27 +26,35 @@
      "integrity": "sha512-hvD8HlzgWkFsCswIHs+wtrIK8HEGC2L8jny3JY00lQ/xwurLxWR+xMYpgjI821INmjEPRFtajFQkniB1HxNBkQ=="
    },
    "@carto/mapnik": {
-      "version": "3.6.2-carto.11",
-      "resolved": "https://registry.npmjs.org/@carto/mapnik/-/mapnik-3.6.2-carto.11.tgz",
-      "integrity": "sha1-8vC8TQBRCAFpJnxccpuQxryTRmE=",
+      "version": "3.6.2-carto.13",
+      "resolved": "https://registry.npmjs.org/@carto/mapnik/-/mapnik-3.6.2-carto.13.tgz",
+      "integrity": "sha512-F1hxZEBpTLvf04ddzf1t1sjNtHBajFPHL6exfzj47lipTDYj1HB1/50ojlg6a+aBE1n5CcCqocHuB7r1cZMxDA==",
      "requires": {
-        "mapnik-vector-tile": "github:cartodb/mapnik-vector-tile#e7ca5471f9e5de81243e6035e70444321fc0a82f",
-        "nan": "2.10.0",
-        "node-pre-gyp": "0.10.0"
+        "mapnik-vector-tile": "github:cartodb/mapnik-vector-tile#v1.6.1-carto.2",
+        "nan": "2.13.2",
+        "node-pre-gyp": "0.12.0"
      },
      "dependencies": {
        "nan": {
-          "version": "2.10.0",
-          "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
-          "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA=="
+          "version": "2.13.2",
+          "resolved": "https://registry.npmjs.org/nan/-/nan-2.13.2.tgz",
+          "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw=="
        }
      }
    },
-    "abaculus": {
-      "version": "github:cartodb/abaculus#3283c523d8bd4bdec78d90166d9c23ff09865ca8",
-      "from": "github:cartodb/abaculus#2.0.3-cdb13",
+    "@carto/tilelive-bridge": {
+      "version": "github:Algunenano/tilelive-bridge#f53c2a681c462fa34ef842a151bceec2746fef9f",
+      "from": "github:Algunenano/tilelive-bridge#mapnik13",
      "requires": {
-        "@carto/mapnik": "3.6.2-carto.11",
+        "@carto/mapnik": "3.6.2-carto.13",
+        "generic-pool": "^3.6.1"
+      }
+    },
+    "abaculus": {
+      "version": "github:Algunenano/abaculus#d390c502130709f547c1c424f22f4c582bf898ec",
+      "from": "github:Algunenano/abaculus#mapnik13",
+      "requires": {
+        "@carto/mapnik": "3.6.2-carto.13",
        "d3-queue": "^2.0.2",
        "sphericalmercator": "1.0.5"
      }
@ -418,7 +426,7 @@
      "integrity": "sha512-1z3Dk9G8KQlNGurbcmGBvNj8DVCh1Keue9uzyyvB6hKOYzBHMxixAMG0D+8nSsA7oQmWUsx/xkZZ5ZxT9toEHA==",
      "requires": {
        "debug": "^3.1.0",
-        "pg": "github:cartodb/node-postgres#5417d7b29b7272ca2e71bb396899ab3f177a9ae6",
+        "pg": "github:cartodb/node-postgres#6.4.2-cdb2",
        "underscore": "~1.6.0"
      }
    },
@ -2042,7 +2050,7 @@
        "carto": "0.16.3",
        "debug": "~3.1.0",
        "generic-pool": "~2.2.0",
-        "millstone": "github:cartodb/millstone#f201885250d2d7ea6c870a1e1060ca6bcfdef186",
+        "millstone": "github:cartodb/millstone#v0.6.17-carto.2",
        "postcss": "~5.2.8",
        "postcss-scss": "0.4.0",
        "postcss-strip-inline-comments": "0.1.5",
@ -2822,22 +2830,27 @@
      "optional": true
    },
    "needle": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz",
-      "integrity": "sha512-HyoqEb4wr/rsoaIDfTH2aVL9nWtQqba2/HvMv+++m8u0dz808MaagKILxtfeSN7QU7nvbQ79zk3vYOJp9zsNEA==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.0.tgz",
+      "integrity": "sha512-QBZu7aAFR0522EyaXZM0FZ9GLpq6lvQ3uq8gteiDUp7wKdy0lSd2hPlgFwVuW1CBkfEs9PfDQsQzZghLs/psdg==",
      "requires": {
-        "debug": "^2.1.2",
+        "debug": "^4.1.0",
        "iconv-lite": "^0.4.4",
        "sax": "^1.2.4"
      },
      "dependencies": {
        "debug": {
-          "version": "2.6.9",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
          "requires": {
-            "ms": "2.0.0"
+            "ms": "^2.1.1"
          }
+        },
+        "ms": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
        }
      }
    },
@ -2864,17 +2877,17 @@
      }
    },
    "node-pre-gyp": {
-      "version": "0.10.0",
-      "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.10.0.tgz",
-      "integrity": "sha512-G7kEonQLRbcA/mOoFoxvlMrw6Q6dPf92+t/l0DFSMuSlDoWaI9JWIyPwK0jyE1bph//CUEL65/Fz1m2vJbmjQQ==",
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz",
+      "integrity": "sha512-4KghwV8vH5k+g2ylT+sLTjy5wmUOb9vPhnM8NHvRf9dHmnW/CndrFXy2aRPaPST6dugXSdHXfeaHQm77PIz/1A==",
      "requires": {
        "detect-libc": "^1.0.2",
        "mkdirp": "^0.5.1",
-        "needle": "^2.2.0",
+        "needle": "^2.2.1",
        "nopt": "^4.0.1",
        "npm-packlist": "^1.1.6",
        "npmlog": "^4.0.2",
-        "rc": "^1.1.7",
+        "rc": "^1.2.7",
        "rimraf": "^2.6.1",
        "semver": "^5.3.0",
        "tar": "^4"
@ -3678,9 +3691,9 @@
      }
    },
    "spdx-license-ids": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.3.tgz",
-      "integrity": "sha512-uBIcIl3Ih6Phe3XHK1NqboJLdGfwr1UN3k6wSD1dZpmPsIkb8AGNbZYJ1fOBk834+Gxy8rpfDxrS6XLEMZMY2g=="
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.4.tgz",
+      "integrity": "sha512-7j8LYJLeY/Yb6ACbQ7F76qy5jHkp0U6jgBfJsk97bwWlVUnUWsAgpyaCvo17h0/RQGnQ036tVDomiwoI4pDkQA=="
    },
    "sphericalmercator": {
      "version": "1.0.5",
@ -3898,12 +3911,81 @@
      "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz",
      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
    },
+    "through2": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/through2/-/through2-0.2.3.tgz",
+      "integrity": "sha1-6zKE2k6jEbbMis42U3SKUqvyWj8=",
+      "requires": {
+        "readable-stream": "~1.1.9",
+        "xtend": "~2.1.1"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "1.1.14",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
+          "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.1",
+            "isarray": "0.0.1",
+            "string_decoder": "~0.10.x"
+          }
+        },
+        "xtend": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.1.2.tgz",
+          "integrity": "sha1-bv7MKk2tjmlixJAbM3znuoe10os=",
+          "requires": {
+            "object-keys": "~0.4.0"
+          }
+        }
+      }
+    },
+    "tilelive": {
+      "version": "5.12.3",
+      "resolved": "https://registry.npmjs.org/tilelive/-/tilelive-5.12.3.tgz",
+      "integrity": "sha1-nIx3DhGUqo01PZodAxR0BDB83r0=",
+      "requires": {
+        "minimist": "~0.2.0",
+        "progress-stream": "~0.5.x",
+        "queue-async": "~1.0.7",
+        "sphericalmercator": "~1.0.1"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "0.2.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.0.tgz",
+          "integrity": "sha1-Tf/lJdriuGTGbC4jxicdev3s784="
+        },
+        "queue-async": {
+          "version": "1.0.7",
+          "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz",
+          "integrity": "sha1-Iq4KHaxKkvW81GNPmTxoKiqBCUU="
+        }
+      }
+    },
+    "tilelive-mapnik": {
+      "version": "github:Algunenano/tilelive-mapnik#b25d2e3a43f70aa9dff378f363c55950761c18d3",
+      "from": "github:Algunenano/tilelive-mapnik#mapnik13",
+      "requires": {
+        "@carto/mapnik": "3.6.2-carto.13",
+        "generic-pool": "^3.6.1",
+        "mime": "2.3.1"
+      },
+      "dependencies": {
+        "mime": {
+          "version": "2.3.1",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz",
+          "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg=="
+        }
+      }
+    },
    "torque.js": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/torque.js/-/torque.js-3.1.0.tgz",
      "integrity": "sha512-xX0BVo/gAy5A2Qx17dbgzjB2cnzuSDwzohdT1hGaKbG0jveL3sKgKE5VyzlN1/l4hP3p6f/m20HvZo3waIawCQ==",
      "requires": {
-        "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c",
+        "carto": "github:cartodb/carto#master",
        "d3": "3.5.17",
        "turbo-carto": "^0.21.1",
        "turf-jenks": "~1.0.1"
@ -4100,23 +4182,24 @@
      "integrity": "sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU="
    },
    "windshaft": {
-      "version": "github:cartodb/windshaft#8692e3c48a51b72aed9411152b53faab8dc05651",
-      "from": "github:cartodb/windshaft#cartonik",
+      "version": "5.1.1",
+      "resolved": "github:algunenano/windshaft#ae05af04504bfa2f9f829ebc35e9cfc00579b310",
      "requires": {
-        "@carto/cartonik": "github:cartodb/cartonik#89dea6126674fef4e2d6800dc8b86e4cd353ae76",
-        "@carto/mapnik": "3.6.2-carto.11",
-        "abaculus": "github:cartodb/abaculus#3283c523d8bd4bdec78d90166d9c23ff09865ca8",
+        "@carto/mapnik": "3.6.2-carto.13",
+        "@carto/tilelive-bridge": "github:Algunenano/tilelive-bridge#mapnik13",
+        "abaculus": "github:Algunenano/abaculus#mapnik13",
        "canvas": "^2.4.1",
-        "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c",
+        "carto": "github:cartodb/carto#0.15.1-cdb5",
        "cartodb-psql": "0.13.1",
        "debug": "3.1.0",
-        "dot": "1.1.2",
        "grainstore": "^2.0.0",
        "queue-async": "1.1.0",
        "redis-mpool": "0.7.0",
        "request": "2.87.0",
        "semver": "5.5.0",
        "sphericalmercator": "1.0.5",
+        "tilelive": "5.12.3",
+        "tilelive-mapnik": "github:Algunenano/tilelive-mapnik#mapnik13",
        "torque.js": "^3.1.0",
        "underscore": "1.6.0"
      }
--- a/package.json
+++ b/package.json
@ -49,7 +49,7 @@
    "step-profiler": "0.3.0",
    "turbo-carto": "0.21.0",
    "underscore": "1.6.0",
-    "windshaft": "github:cartodb/windshaft#cartonik",
+    "windshaft": "5.1.1",
    "yargs": "11.1.0"
  },
  "devDependencies": {
--- a/test/acceptance/cluster.js
+++ b/test/acceptance/cluster.js
@ -555,7 +555,14 @@ describe('cluster', function () {
                        return done(err);
                    }

-                    assert.deepStrictEqual(body.rows, expected);
+                    const sort_f = ((a, b) => {
+                        return (a._cdb_feature_count < b._cdb_feature_count) ||
+                               (a._cdb_feature_count === b._cdb_feature_count &&
+                                    (a.type < b.type ||
+                                     a.type === b.type && a.max_value < b.max_value));
+                    });
+
+                    assert.deepStrictEqual(body.rows.sort(sort_f), expected.sort(sort_f));

                    testClient.drain(done);
                });
--- a/test/acceptance/named_maps_static_view.js
+++ b/test/acceptance/named_maps_static_view.js
@ -106,7 +106,15 @@ describe('named maps static view', function() {
            }
            getStaticMap(function(err, img) {
                assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('estimated'), PNG_IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('estimated'), PNG_IMAGE_TOLERANCE,  err => {
+                    if (err) {
+                        assert.imageIsSimilarToFile(img, previewFixture('estimated-proj5'), PNG_IMAGE_TOLERANCE, done);
+                    }
+                    else
+                    {
+                        done();
+                    }
+                });
            });
        });
    });
--- a/test/fixtures/previews/populated_places_simple_reduced-estimated-proj5.png
+++ b/test/fixtures/previews/populated_places_simple_reduced-estimated-proj5.png
--- a/test/support/prepare_db.sh
+++ b/test/support/prepare_db.sh
@ -13,7 +13,6 @@
 PREPARE_REDIS=yes
 PREPARE_PGSQL=yes
 DOWNLOAD_SQL_FILES=yes
-PG_PARALLEL=$(pg_config --version | (awk '{$2*=1000; if ($2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))

 while [ -n "$1" ]; do
  OPTION=$(echo "$1" | tr -d '[:space:]')