diff --git a/.travis.yml b/.travis.yml index a7a3cbf..c28e5ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,12 @@ env: matrix: include: - - env: PGSQL_VERSION=9.5 POSTGIS_VERSION=2.2 - env: PGSQL_VERSION=9.6 POSTGIS_VERSION=2.3 - env: PGSQL_VERSION=10 POSTGIS_VERSION=2.4 before_install: - sudo bash $TRAVIS_BUILD_DIR/scripts/ci/install_postgres.sh + - sudo make clean-all install: - sudo make install diff --git a/scripts/ci/install_postgres.sh b/scripts/ci/install_postgres.sh index 9ea6676..1b3929f 100644 --- a/scripts/ci/install_postgres.sh +++ b/scripts/ci/install_postgres.sh @@ -27,7 +27,7 @@ done apt-get -y autoremove # Install PostgreSQL -apt-get -y install postgresql-${PGSQL_VERSION} postgresql-${PGSQL_VERSION}-postgis-${POSTGIS_VERSION} postgresql-server-dev-${PGSQL_VERSION} +apt-get -y install postgresql-${PGSQL_VERSION} postgresql-${PGSQL_VERSION}-postgis-${POSTGIS_VERSION} postgresql-server-dev-${PGSQL_VERSION} postgresql-plpython-${PGSQL_VERSION} # Configure it to accept local connections from postgres echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" > /etc/postgresql/${PGSQL_VERSION}/main/pg_hba.conf diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql index aa24851..8a6185e 100644 --- a/src/pg/sql/45_observatory_mvt.sql +++ b/src/pg/sql/45_observatory_mvt.sql @@ -218,16 +218,65 @@ BEGIN END $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(z INTEGER, x INTEGER, y INTEGER, -geography_level TEXT, -do_measurements TEXT[], -mastercard_measurements TEXT[], -shoreline_clipped BOOLEAN DEFAULT True, -optimize_clipping BOOLEAN DEFAULT False, -simplify_geometries BOOLEAN DEFAULT False, -area_normalized BOOLEAN DEFAULT False, -mastercard_category TEXT DEFAULT 'Total Retail', -extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT True) +DROP TABLE IF EXISTS cdb_observatory.OBS_CachedMeta; +CREATE TABLE cdb_observatory.OBS_CachedMeta( + z INTEGER, + parameters TEXT, + num_timespans INTEGER, + num_scores INTEGER, + num_target_geoms INTEGER, + result JSON, + PRIMARY KEY (z, parameters, num_timespans, num_scores, num_target_geoms) +); + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_RetrieveMeta( + zoom INTEGER, + geom geometry(Geometry, 4326), + getmeta_parameters JSON, + num_timespan_options INTEGER DEFAULT NULL, + num_score_options INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON +AS $$ +DECLARE + result JSON; +BEGIN + SELECT c.result + INTO result + FROM cdb_observatory.OBS_CachedMeta c + WHERE c.z = zoom + AND c.parameters = getmeta_parameters::TEXT + AND c.num_timespans = num_timespan_options + AND c.num_scores = num_score_options + AND c.num_target_geoms = target_geoms; + + IF result IS NULL THEN + result := cdb_observatory.obs_getmeta(geom, getmeta_parameters, num_timespan_options, num_score_options, target_geoms); + + INSERT INTO cdb_observatory.OBS_CachedMeta(z, parameters, num_timespans, num_scores, num_target_geoms, result) + SELECT zoom, getmeta_parameters::TEXT, num_timespan_options, num_score_options, target_geoms, result + ON CONFLICT (z, parameters, num_timespans, num_scores, num_target_geoms) + DO UPDATE SET result = EXCLUDED.result; + END IF; + + return result; +END +$$ LANGUAGE plpgsql PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT( + z INTEGER, x INTEGER, y INTEGER, + geography_level TEXT, + do_measurements TEXT[], + mastercard_measurements TEXT[], + use_meta_cache BOOLEAN DEFAULT True, + shoreline_clipped BOOLEAN DEFAULT True, + optimize_clipping BOOLEAN DEFAULT False, + simplify_geometries BOOLEAN DEFAULT False, + area_normalized BOOLEAN DEFAULT False, + mastercard_category TEXT DEFAULT 'Total Retail', + extent INTEGER DEFAULT 4096, + buf INTEGER DEFAULT 256, + clip_geom BOOLEAN DEFAULT True) RETURNS TABLE ( mvtgeom GEOMETRY, mvtdata JSONB @@ -317,7 +366,11 @@ BEGIN END LOOP; getmeta_parameters := substring(getmeta_parameters from 1 for length(getmeta_parameters) - 1) || ' ]'; - meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer); + IF use_meta_cache THEN + meta := cdb_observatory.OBS_RetrieveMeta(z, geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer); + ELSE + meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer); + END IF; IF meta IS NOT NULL THEN SELECT array_agg(distinct 'observatory.'||numer_tablename) numer_tablenames, diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out index d6e253f..213e9c2 100644 --- a/src/pg/test/expected/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -1,5 +1,6 @@ -- Install dependencies CREATE EXTENSION postgis; +CREATE LANGUAGE plpythonu; -- Install the extension CREATE EXTENSION observatory VERSION 'dev'; \i test/fixtures/load_fixtures.sql diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql index 5c2dbf3..fe536ca 100644 --- a/src/pg/test/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -1,5 +1,6 @@ -- Install dependencies CREATE EXTENSION postgis; +CREATE LANGUAGE plpythonu; -- Install the extension CREATE EXTENSION observatory VERSION 'dev';