Merge pull request #1056 from CartoDB/nodejs-10

Support Nodejs 10 LTS

.travis.yml

@@ -5,7 +5,13 @@ jobs:
         - docker
       language: generic
       before_install: docker pull carto/nodejs6-xenial-pg101:postgis-2.4.4.5
-      script: npm run docker-test
+      script: npm run docker-test -- nodejs6
+    - sudo: required
+      services:
+        - docker
+      language: generic
+      before_install: docker pull carto/nodejs10-xenial-pg101:postgis-2.4.4.5
+      script: npm run docker-test -- nodejs10
     - dist: precise
       addons:
         postgresql: "9.5"

NEWS.md

@@ -4,15 +4,23 @@
 Released 2018-mm-dd
 
 New features
+- Suport Node.js 10
+- Configure travis to run docker tests against Node.js 6 & 10 versions
 - Aggregation time dimensions
 - Update sample configurations to use PostGIS to generate MVT's by default (as in production)
-- Upgrades Windshaft to [4.11.4](https://github.com/CartoDB/Windshaft/blob/4.11.4/NEWS.md#version-4114)
+- Upgrades Windshaft to [4.12.0](https://github.com/CartoDB/Windshaft/blob/4.12.0/NEWS.md#version-4120)
   - `pg-mvt`: Use `query-rewriter` to compose the query to render a MVT tile. If not defined, it will use a Default Query Rewriter.
   - `pg-mvt`: Fix bug while building query and there is no columns defined for the layer.
   - `pg-mvt`: Accept trailing semicolon in input queries.
   - `Renderer Cache Entry`: Do not throw errors for integrity checks.
   - Fix bug when releasing the renderer cache entry in some scenarios.
-  - Upgrade grainstore to [1.9.1](https://github.com/CartoDB/grainstore/releases/tag/1.9.1).
+  - Upgrade grainstore to [1.10.0](https://github.com/CartoDB/grainstore/releases/tag/1.10.0)
+- Upgrade cartodb-redis to [2.1.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/2.1.0)
+- Upgrade cartodb-query-tables to [0.4.0](https://github.com/CartoDB/node-cartodb-query-tables/releases/tag/0.4.0)
+- Upgrade cartodb-psql to [0.13.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.13.0)
+- Upgrade turbo-carto to [0.21.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.21.0)
+- Upgrade camshaft to [0.63.0](https://github.com/CartoDB/camshaft/releases/tag/0.63.0)
+- Upgrade redis-mpool to [0.7.0](https://github.com/CartoDB/node-redis-mpool/releases/tag/0.7.0)
 
 Bug Fixes:
 - Prevent from uncaught exception: Range filter Error from camshaft when getting analysis query.

docker-bash.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run  -it -v `pwd`:/srv carto/${1:-nodejs10-xenial-pg101:postgis-2.4.4.5} bash

docker-test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 [nodejs6|nodejs10]"
+    exit 1
+}
+
+echo "$0 $1"
+
+NODEJS_VERSION=${1-nodejs10}
+
+if [ "$NODEJS_VERSION" = "nodejs10" ];
+then
+    DOCKER='nodejs10-xenial-pg101:postgis-2.4.4.5'
+elif [ "$NODEJS_VERSION" = "nodejs6" ];
+then
+    DOCKER='nodejs6-xenial-pg101:postgis-2.4.4.5'
+else
+    usage
+fi
+
+docker run -v `pwd`:/srv carto/${DOCKER} bash run_tests_docker.sh ${NODEJS_VERSION} && \
+    docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v

docker/Dockerfile-nodejs10-xenial-pg101:postgis-2.4.5.5

@@ -0,0 +1,88 @@
+FROM ubuntu:xenial
+
+# Use UTF8 to avoid encoding problems with pgsql
+ENV LANG C.UTF-8
+ENV NPROCS 1
+ENV JOBS 1
+ENV CXX g++-4.9
+ENV PGUSER postgres
+
+# Add external repos
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        curl \
+        software-properties-common \
+        locales \
+    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+    && add-apt-repository -y ppa:cartodb/postgresql-10 \
+    && add-apt-repository -y ppa:cartodb/gis \
+    && curl -sL https://deb.nodesource.com/setup_10.x | bash \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        g++-4.9 \
+        gcc-4.9 \
+        git \
+        libcairo2-dev \
+        libgdal-dev \
+        libgdal1i \
+        libgdal20 \
+        libgeos-dev \
+        libgif-dev \
+        libjpeg8-dev \
+        libjson-c-dev \
+        libpango1.0-dev \
+        libpixman-1-dev \
+        libproj-dev \
+        libprotobuf-c-dev \
+        libxml2-dev \
+        gdal-bin \
+        make \
+        nodejs \
+        protobuf-c-compiler \
+        pkg-config \
+        wget \
+        zip \
+        postgresql-10 \
+        postgresql-10-plproxy \
+        postgis=2.4.4.5+carto-1 \
+        postgresql-10-postgis-2.4=2.4.4.5+carto-1 \
+        postgresql-10-postgis-2.4-scripts=2.4.4.5+carto-1 \
+        postgresql-10-postgis-scripts=2.4.4.5+carto-1 \
+        postgresql-client-10 \
+        postgresql-client-common \
+        postgresql-common \
+        postgresql-contrib \
+        postgresql-plpython-10 \
+        postgresql-server-dev-10 \
+    && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
+    && tar xvzf redis-4.0.8.tar.gz \
+    && cd redis-4.0.8 \
+    && make \
+    && make install \
+    && cd .. \
+    && rm redis-4.0.8.tar.gz \
+    && rm -R redis-4.0.8 \
+    && apt-get purge -y wget protobuf-c-compiler \
+    && apt-get autoremove -y
+
+# Configure PostgreSQL
+RUN set -ex \
+    && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
+    && echo "local     all       all                     trust" >  /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       0.0.0.0/0     trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       ::1/128       trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && /etc/init.d/postgresql start \
+    && createdb template_postgis \
+    && createuser publicuser \
+    && psql -c "CREATE EXTENSION postgis" template_postgis \
+    && /etc/init.d/postgresql stop
+
+WORKDIR /srv
+EXPOSE 5858
+
+CMD /etc/init.d/postgresql start

docker/reference.md

@@ -11,13 +11,13 @@ https://hub.docker.com/r/carto/
 
 ## Update image
 - Edit the docker image file with your desired changes
-- Build image: 
+- Build image:
   - `docker build -t carto/IMAGE -f docker/DOCKER_FILE docker/`
 
 - Upload to docker hub:
-  - Login into docker hub: 
+  - Login into docker hub:
     - `docker login`
-  - Create tag: 
+  - Create tag:
     - `docker tag carto/IMAGE carto/IMAGE`
-  - Upload: 
+  - Upload:
     - `docker push carto/IMAGE`

package.json

@@ -27,28 +27,28 @@
     "@carto/fqdn-sync": "0.2.2",
     "basic-auth": "2.0.0",
     "body-parser": "1.18.3",
-    "camshaft": "0.62.4",
-    "cartodb-psql": "0.11.0",
-    "cartodb-query-tables": "0.3.0",
-    "cartodb-redis": "2.0.2",
+    "camshaft": "0.63.0",
+    "cartodb-psql": "0.13.0",
+    "cartodb-query-tables": "0.4.0",
+    "cartodb-redis": "2.1.0",
     "debug": "3.1.0",
     "dot": "1.1.2",
     "express": "4.16.3",
     "fastly-purge": "1.0.1",
     "glob": "7.1.2",
-    "log4js": "cartodb/log4js-node#cdb",
+    "log4js": "github:cartodb/log4js-node#cdb",
     "lru-cache": "4.1.3",
     "lzma": "2.3.2",
     "node-statsd": "0.1.1",
     "on-headers": "1.0.1",
     "queue-async": "1.1.0",
-    "redis-mpool": "0.6.0",
+    "redis-mpool": "0.7.0",
     "request": "2.87.0",
     "semver": "5.5.0",
     "step-profiler": "0.3.0",
-    "turbo-carto": "0.20.4",
+    "turbo-carto": "0.21.0",
     "underscore": "1.6.0",
-    "windshaft": "4.11.5",
+    "windshaft": "4.12.0",
     "yargs": "11.1.0"
   },
   "devDependencies": {
@@ -66,8 +66,8 @@
     "preinstall": "make pre-install",
     "test": "make test-all",
     "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn",
-    "docker-test": "docker run -v `pwd`:/srv carto/nodejs6-xenial-pg101:postgis-2.4.4.5 bash run_tests_docker.sh && docker ps --filter status=dead --filter status=exited -aq | xargs -r docker rm -v",
-    "docker-bash": "docker run  -it -v `pwd`:/srv carto/nodejs6-xenial-pg101 bash"
+    "docker-test": "./docker-test.sh",
+    "docker-bash": "./docker-bash.sh"
   },
   "engines": {
     "node": ">=6.9",

run_tests_docker.sh

@@ -1,9 +1,40 @@
+#!/bin/bash
+
+usage() {
+    /etc/init.d/postgresql stop
+    echo "Usage: $0 [nodejs10|nodejs6]"
+    exit 1
+}
+
+echo "$0 $1"
+
 # start PostgreSQL
 /etc/init.d/postgresql start
 
+echo "Node.js version:"
+node -v
+
 # install dependencies
-npm install -g yarn@0.27.5
-yarn
+NODEJS_VERSION=${1-nodejs10}
+
+if [ "$NODEJS_VERSION" = "nodejs10" ];
+then
+    echo "npm version on install:"
+    npm -v
+    npm ci
+    npm ls
+elif [ "$NODEJS_VERSION" = "nodejs6" ];
+then
+    npm install -g yarn@0.27.5
+    echo "yarn version on install:"
+    yarn --version
+    yarn
+else
+    usage
+fi
 
 # run tests
+echo "npm version on tests:"
+npm -v
+
 npm test

test/acceptance/user-database-timeout-limit.js

@@ -24,11 +24,23 @@ const validationPointSleepSql = `
         2 val
 `;
 
+const cartoCSSPoints = () => `
+// cache buster: ${Date.now()}
+#layer{
+  marker-placement: point;
+  marker-allow-overlap: true;
+  marker-line-opacity: 0.2;
+  marker-line-width: 0.5;
+  marker-opacity: 1;
+  marker-width: 5;
+  marker-fill: red;
+}`;
+
 const createMapConfig = ({
     version = '1.6.0',
     type = 'cartodb',
     sql = pointSleepSql,
-    cartocss = TestClient.CARTOCSS.POINTS,
+    cartocss = cartoCSSPoints(),
     cartocss_version = '2.3.0',
     interactivity = 'cartodb_id',
     countBy = 'cartodb_id',

test/support/test-client.js

@@ -1267,6 +1267,11 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb
     const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 });
     const publicuser = global.environment.postgres.user;
 
+    // IMPORTANT: node-postgres uses internallly a singleton, to refresh all pull connections
+    // you need to ensure that your dependency tree has only one working version of `cartodb-psql` & `node-postgres`
+    // if not, test using this function cannot ensure that all connections have the new settings (STATEMENT_TIMEOUT)
+    //
+    // TODO: upgrade to node-postgres@7.x
     const psql = new PSQL({
         user: 'postgres',
         dbname: dbname,
@@ -1274,9 +1279,6 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb
         port: global.environment.postgres.port
     });
 
-    // we need to guarantee all new connections have the new settings
-    psql.end();
-
     step(
         function configureTimeouts () {
             const timeoutSQLs = [
@@ -1289,7 +1291,10 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb
 
             timeoutSQLs.forEach(sql => psql.query(sql, group()));
         },
-        callback
+        // we need to guarantee all new connections have the new settings
+        function refreshPoolConnection () {
+            psql.end(() => callback());
+        }
     );
 };