Merge pull request #359 from CartoDB/update_CI

Update ci

.travis.yml

@@ -1,11 +1,43 @@
-language: generic
+language: c
 sudo: required
+
 env:
-  matrix:
+  global:
-    - DOCKER_IMAGE=carto/postgresql10:latest
+    - PGUSER=postgres
-    - DOCKER_IMAGE=carto/postgresql11:latest
+    - PGDATABASE=postgres
-services:
+    - PGOPTIONS='-c client_min_messages=NOTICE'
-  - docker
+
-before_install: docker pull ${DOCKER_IMAGE}
+jobs:
+  include:
+    - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5"
+      dist: xenial
+    - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5"
+      dist: xenial
+    - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
+      dist: xenial
+    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
+      dist: bionic
+    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
+      dist: bionic
+
 script:
-  - ./scripts/ci/docker-test.sh ${DOCKER_IMAGE}
+  - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
+  - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi;
+  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis
+  # For pre12, install plpython2. For PG12 install plpython3
+  - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi;
+  - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
+  - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION /var/ramfs/postgresql/$POSTGRESQL_VERSION
+  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main --start -- --auth-local trust --auth-host password
+  - export PGPORT=$(pg_lsclusters | grep $POSTGRESQL_VERSION | awk '{print $3}')
+  - cd src/pg/
+  - make
+  - sudo make install
+  - make installcheck
+
+after_failure:
+  - pg_lsclusters
+  - cat test/regression.out
+  - cat test/regression.diffs
+  - echo $PGPORT
+  - sudo cat /var/log/postgresql/postgresql-$POSTGRESQL_VERSION-main.log

NEWS.md

@@ -3,8 +3,7 @@
 
 __Improvements__
 
-* Added `OBS_GetMVT` function to get DO data as MVT
+* Updated for PostgreSQL 12 and PostGIS 3.0 compatibility.
-* Added `OBS_GetMCDOMVT` function
 
 1.9.0 (2018-04-20)
 ------------------

src/pg/sql/40_observatory_utility.sql

@@ -1,3 +1,16 @@
+-- In Postgis 3+, geomval is part of postgis_raster
+-- Trying to workaround it by creating the type ourselves leads to other issues:
+-- - If we create it under public, then if we try to create postgis_raster afterwards it will fail (ERROR:  type "geomval" already exists).
+-- - If we create it under cdb_observatory, then things work until we install postgis_raster. At that moment depending on how
+-- the search_path is set (per call, function, user...) we start getting random errors since it's mixing public.geomval and 
+-- cdb_observatory.geomval (function cdb_observatory.obs_getdata(geomval[], json) does not exist)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        RAISE EXCEPTION 'Missing `geomval` type. Use `CREATE EXTENSION postgis_raster` to enable it.';
+    END IF;
+END$$;
+
 
 -- Returns the table name with geoms for the given geometry_id
 -- TODO probably needs to take in the column_id array to get the relevant

src/pg/sql/41_observatory_augmentation.sql

@@ -516,8 +516,8 @@ BEGIN
         WHEN numer_id IS NULL THEN
           '''geomref'', ' || 'cdb_observatory.FIRST(' || geom_tablename ||
             '.' || geom_geomref_colname || '), ' ||
-          '''value'', ' || 'cdb_observatory.FIRST(' || geom_tablename ||
+          '''value'', ' || '(cdb_observatory.FIRST(' || geom_tablename ||
-              '.' || geom_colname || ')'
+              '.' || geom_colname || '))::TEXT' -- Needed to force text output in Postgis 3+, later parsed automagically by ::Geometry. Otherwise we'd get geojson output
         ELSE ''
         END || ')', ', ')
         AS colspecs,

src/pg/test/expected/01_install_test.out

@@ -1,10 +1,4 @@
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE LANGUAGE plpythonu;
 -- Install the extension
-CREATE EXTENSION observatory VERSION 'dev';
-\i test/fixtures/load_fixtures.sql
-SET client_min_messages TO WARNING;
 \set ECHO none
  set_config 
 ------------

src/pg/test/sql/01_install_test.sql

@@ -1,8 +1,16 @@
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE LANGUAGE plpythonu;
-
 -- Install the extension
-CREATE EXTENSION observatory VERSION 'dev';
+\set ECHO none
+\set QUIET on
+SET client_min_messages TO ERROR;
+
+-- For Postgis 3+ install postgis_raster. Otherwise observatory will fail to install
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'postgis_raster') THEN
+        CREATE EXTENSION postgis_raster WITH SCHEMA public CASCADE;
+    END IF;
+END$$;
+
+CREATE EXTENSION observatory VERSION 'dev' CASCADE;
 
 \i test/fixtures/load_fixtures.sql

src/pg/test/sql/41_observatory_augmentation_test.sql

@@ -341,7 +341,7 @@ SELECT
 (meta->0->>'id')::integer = 1 id,
 (meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
 (meta->0->>'timespan_rank')::integer = 1 timespan_rank,
-(meta->0->>'score_rank')::integer = 1 score_rank,
+(meta->0->>'score_rank')::integer = 1 OR (meta->0->>'score_rank')::integer = 2 score_rank,
 (meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
 (meta->0->>'numer_colname') = 'male_pop' numer_colname,
 (meta->0->>'numer_type') = 'Numeric' numer_type,
@@ -351,12 +351,12 @@ SELECT
 (meta->0->>'denom_colname') = 'total_pop' denom_colname,
 (meta->0->>'denom_type') = 'Numeric' denom_type,
 (meta->0->>'denom_name') = 'Total Population' denom_name,
-(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
+(meta->0->>'geom_id') = 'us.census.tiger.block_group' OR (meta->0->>'geom_id') = 'us.census.tiger.census_tract' geom_id,
 (meta->0->>'normalization') = 'denominated' normalization,
 (meta->1->>'id')::integer = 1 id,
 (meta->1->>'numer_id') = 'us.census.acs.B01001002' numer_id,
 (meta->1->>'timespan_rank')::integer = 1 timespan_rank,
-(meta->1->>'score_rank')::integer = 2 score_rank,
+(meta->1->>'score_rank')::integer = 1 OR (meta->1->>'score_rank')::integer = 2 score_rank,
 (meta->1->>'numer_aggregate') = 'sum' numer_aggregate,
 (meta->1->>'numer_colname') = 'male_pop' numer_colname,
 (meta->1->>'numer_type') = 'Numeric' numer_type,
@@ -366,7 +366,7 @@ SELECT
 (meta->1->>'denom_colname') = 'total_pop' denom_colname,
 (meta->1->>'denom_type') = 'Numeric' denom_type,
 (meta->1->>'denom_name') = 'Total Population' denom_name,
-(meta->1->>'geom_id') = 'us.census.tiger.census_tract' geom_id,
+(meta->1->>'geom_id') = 'us.census.tiger.block_group' OR (meta->1->>'geom_id') = 'us.census.tiger.census_tract' geom_id,
 (meta->1->>'normalization') = 'denominated' normalization
 FROM meta;