From dd907ac2bc85d1dab1d0ff97b643fdb5da5fa502 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 24 Feb 2020 16:12:27 +0100 Subject: [PATCH 01/21] Using @@plpythonu@@ variable PG12 migration #6237 --- client/Makefile | 34 ++++++++++++++++--- client/sql/20_table_augmentation.sql | 4 +-- client/test/expected/00_installation_test.out | 2 +- client/test/sql/00_installation_test.sql | 2 +- .../extension/cdb_dataservices_server.control | 2 +- server/extension/sql/100_routing_helper.sql | 6 ++-- .../sql/105_route_between_points.sql | 4 +-- server/extension/sql/10_redis_helper.sql | 2 +- .../sql/110_data_observatory_augmentation.sql | 30 ++++++++-------- .../sql/115_data_observatory_exploration.sql | 4 +-- .../sql/120_data_observatory_geometries.sql | 14 ++++---- .../125_data_observatory_table_augment.sql | 2 +- server/extension/sql/15_config_helper.sql | 12 +++---- server/extension/sql/200_quotas.sql | 4 +-- server/extension/sql/20_geocode_street.sql | 26 +++++++------- server/extension/sql/210_rates.sql | 8 ++--- .../extension/sql/21_bulk_geocode_street.sql | 12 +++---- server/extension/sql/30_admin0.sql | 2 +- server/extension/sql/40_admin1.sql | 4 +-- server/extension/sql/50_namedplaces.sql | 8 ++--- server/extension/sql/60_postalcodes.sql | 8 ++--- server/extension/sql/70_ips.sql | 2 +- server/extension/sql/80_isolines_helper.sql | 14 ++++---- server/extension/sql/85_isodistance.sql | 10 +++--- server/extension/sql/90_isochrone.sql | 10 +++--- .../test/expected/00_install_test.out | 2 +- server/extension/test/sql/00_install_test.sql | 2 +- .../test/sql/366_empty_table_test.sql | 2 +- 28 files changed, 129 insertions(+), 103 deletions(-) diff --git a/client/Makefile b/client/Makefile index 50c7102..d292cb9 100644 --- a/client/Makefile +++ b/client/Makefile @@ -14,6 +14,14 @@ AWK = awk PG_CONFIG = pg_config PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) +# PG12 compatibility +PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') +PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) +PLPYTHONU := plpythonu +ifeq ($(PG_12_GE), true) +PLPYTHONU := plpython3u +endif + # OLD_VERSIONS = $(wildcard old_versions/*.sql) # DATA = $(NEW_EXTENSION_ARTIFACT) \ # $(OLD_VERSIONS) \ @@ -25,9 +33,9 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) SOURCES_DATA_DIR = sql/ - REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) -TEST_DIR = test/ +REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out)))) +TEST_DIR = test_re REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres' # DATA is a special variable used by postgres build infrastructure @@ -54,7 +62,8 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES) $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ - cat $(SOURCES_DATA_DIR)/*.sql >> $@ + cat $(SOURCES_DATA_DIR)/*.sql | \ + $(SED) -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@ ifeq ($(PG_PARALLEL), 0) # Remove PARALLEL in aggregates and functions $(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX)) @@ -86,6 +95,7 @@ release: $(EXTENSION).control $(SOURCES_DATA) devclean: rm -f $(NEW_EXTENSION_ARTIFACT) rm -f $(GENERATED_SQL_FILES) + rm -rf $(TEST_DIR) # If needed remove PARALLEL tags from the release files release_remove_parallel_deploy: @@ -97,9 +107,25 @@ ifeq ($(PG_PARALLEL), 0) done endif +# TODO +prepare_tests: + mkdir -p $(TEST_DIR) + mkdir -p $(TEST_DIR)/expected + mkdir -p $(TEST_DIR)/sql + for f in $(REGRESS); do \ + cat test/sql/$${f}.sql | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + >> $(TEST_DIR)/sql/$${f}.sql; \ + done + for f in $(REGRESS_EXPEC); do \ + cat test/expected/$${f}.out | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + >> $(TEST_DIR)/expected/$${f}.out; \ + done + # Install the current release into the PostgreSQL extensions directory deploy: release_remove_parallel_deploy $(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/' -install: deploy +install: prepare_tests deploy diff --git a/client/sql/20_table_augmentation.sql b/client/sql/20_table_augmentation.sql index 3ab46a9..7dd48e8 100644 --- a/client/sql/20_table_augmentation.sql +++ b/client/sql/20_table_augmentation.sql @@ -138,7 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeas ) return True -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( username text, @@ -231,7 +231,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMea fdw_server=plpy.quote_literal(server_name))) return True -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable( username text, diff --git a/client/test/expected/00_installation_test.out b/client/test/expected/00_installation_test.out index 650b141..fb6acde 100644 --- a/client/test/expected/00_installation_test.out +++ b/client/test/expected/00_installation_test.out @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION cartodb; CREATE EXTENSION plproxy; -- Install the extension diff --git a/client/test/sql/00_installation_test.sql b/client/test/sql/00_installation_test.sql index 49eab80..89afd26 100644 --- a/client/test/sql/00_installation_test.sql +++ b/client/test/sql/00_installation_test.sql @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION cartodb; CREATE EXTENSION plproxy; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 9788bda..3e18400 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' default_version = '0.37.0' -requires = 'plpythonu, plproxy, postgis, cdb_geocoder' +requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index e5f18ad..eeafba5 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -69,7 +69,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate Mapbox routing') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( username TEXT, @@ -137,7 +137,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate TomTom routing') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( username TEXT, @@ -198,4 +198,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate mapzen routing') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 537feb7..e5d4ae6 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -37,7 +37,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( @@ -76,4 +76,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/10_redis_helper.sql b/server/extension/sql/10_redis_helper.sql index 1ba0ca4..2b39b03 100644 --- a/server/extension/sql/10_redis_helper.sql +++ b/server/extension/sql/10_redis_helper.sql @@ -15,4 +15,4 @@ RETURNS boolean AS $$ 'redis_metrics_connection': redis_metrics_connection, } return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index 95f8f20..738262f 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -14,7 +14,7 @@ RETURNS text AS $$ user_obs_config = GD["user_obs_config_{0}".format(username)] return user_obs_config.connection_str -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( username TEXT, @@ -68,7 +68,7 @@ RETURNS json AS $$ raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( username TEXT, @@ -125,7 +125,7 @@ RETURNS SETOF JSON AS $$ raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( username TEXT, @@ -177,7 +177,7 @@ RETURNS json AS $$ raise Exception('Error trying to obs_get_segment_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( username TEXT, @@ -232,7 +232,7 @@ RETURNS SETOF JSON AS $$ raise Exception('Error trying to OBS_GetSegmentSnapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( username TEXT, @@ -289,7 +289,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetMeasure') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( username TEXT, @@ -344,7 +344,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetCategory') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( username TEXT, @@ -401,7 +401,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetUSCensusMeasure') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( username TEXT, @@ -456,7 +456,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetUSCensusCategory') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( username TEXT, @@ -511,7 +511,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetPopulation') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( username TEXT, @@ -566,7 +566,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( username TEXT, @@ -627,7 +627,7 @@ RETURNS TABLE ( raise Exception('Error trying to OBS_GetData') finally: quota_service.increment_total_service_use(len(geomvals)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( username TEXT, @@ -688,7 +688,7 @@ RETURNS TABLE ( raise Exception('Error trying to OBS_GetData') finally: quota_service.increment_total_service_use(len(geomrefs)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( username TEXT, @@ -736,7 +736,7 @@ RETURNS JSON AS $$ import sys logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to OBS_GetMeta') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( username TEXT, @@ -782,4 +782,4 @@ RETURNS TABLE(valid boolean, errors text[]) AS $$ import sys logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to OBS_MetadataValidation') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index fcdc743..faab9a7 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -55,7 +55,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) raise Exception('Error trying to OBS_Search') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( username TEXT, @@ -113,4 +113,4 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index c0b6290..9524378 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -49,7 +49,7 @@ RETURNS geometry(Geometry, 4326) AS $$ raise Exception('Error trying to OBS_GetBoundary') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( username TEXT, @@ -102,7 +102,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetBoundaryId') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( username TEXT, @@ -155,7 +155,7 @@ RETURNS geometry(Geometry, 4326) AS $$ raise Exception('Error trying to OBS_GetBoundaryById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( username TEXT, @@ -215,7 +215,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetBoundariesByGeometry') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( username TEXT, @@ -277,7 +277,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( username TEXT, @@ -337,7 +337,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetPointsByGeometry') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( username TEXT, @@ -399,4 +399,4 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetPointsByPointAndRadius') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/125_data_observatory_table_augment.sql b/server/extension/sql/125_data_observatory_table_augment.sql index e64121e..6e1bad3 100644 --- a/server/extension/sql/125_data_observatory_table_augment.sql +++ b/server/extension/sql/125_data_observatory_table_augment.sql @@ -8,7 +8,7 @@ RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) )[0] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 6e4d884..49f7735 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -8,7 +8,7 @@ RETURNS boolean AS $$ logger_config = LoggerConfig(plpy) GD[cache_key] = logger_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; -- This is done in order to avoid an undesired depedency on cartodb extension CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) @@ -46,7 +46,7 @@ RETURNS boolean AS $$ geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) GD[cache_key] = geocoder_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ @@ -60,7 +60,7 @@ RETURNS boolean AS $$ geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) GD[cache_key] = geocoder_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) RETURNS boolean AS $$ @@ -74,7 +74,7 @@ RETURNS boolean AS $$ isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = isolines_routing_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) RETURNS boolean AS $$ @@ -88,7 +88,7 @@ RETURNS boolean AS $$ routing_config = RoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = routing_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) RETURNS boolean AS $$ @@ -102,4 +102,4 @@ RETURNS boolean AS $$ obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) GD[cache_key] = obs_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 973d1ce..23ee403 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -106,7 +106,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ ret += [[service, monthly_quota, used_quota, soft_limit, provider]] return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( @@ -136,7 +136,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 76c01be..00674f3 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ else: raise Exception('Requested geocoder is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) @@ -52,7 +52,7 @@ RETURNS Geometry AS $$ else: raise Exception('Here geocoder is not available for your account.') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -67,7 +67,7 @@ RETURNS Geometry AS $$ else: raise Exception('Google geocoder is not available for your account.') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -80,7 +80,7 @@ RETURNS Geometry AS $$ mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -93,7 +93,7 @@ RETURNS Geometry AS $$ mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -106,7 +106,7 @@ RETURNS Geometry AS $$ tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -119,7 +119,7 @@ RETURNS Geometry AS $$ geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -152,7 +152,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using here maps') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -184,7 +184,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using google maps') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -225,7 +225,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using mapzen') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -271,7 +271,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using Mapbox') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -317,7 +317,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using TomTom') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -363,4 +363,4 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using Geocodio') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/210_rates.sql b/server/extension/sql/210_rates.sql index 6fa97d7..f83f0da 100644 --- a/server/extension/sql/210_rates.sql +++ b/server/extension/sql/210_rates.sql @@ -16,7 +16,7 @@ RETURNS JSON AS $$ return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) else: return None -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( username TEXT, @@ -40,7 +40,7 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_user_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( username TEXT, @@ -64,7 +64,7 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_org_rate_limits(config) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( username TEXT, @@ -88,4 +88,4 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_server_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 8e0b5ea..e41e67e 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -40,7 +40,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) return plpy.execute(plan, [username, orgname, searches]) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -51,7 +51,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = LegacyServiceManager('geocoder', username, orgname, GD) geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -62,7 +62,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = LegacyServiceManager('geocoder', username, orgname, GD) geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -79,7 +79,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -96,7 +96,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -113,4 +113,4 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD) geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index c5d8c8e..dead116 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -35,7 +35,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin0 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index c10e3b2..75ac7dd 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) @@ -74,7 +74,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 932fc78..0e9f14e 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -16,7 +16,7 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) @@ -56,7 +56,7 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ @@ -101,7 +101,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode namedplace point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index d82f42e..fcd0e53 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -35,7 +35,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) RETURNS Geometry AS $$ @@ -77,7 +77,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ @@ -119,7 +119,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) RETURNS Geometry AS $$ @@ -162,7 +162,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 0f41c5a..8fa025f 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -35,7 +35,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index a259676..af65951 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -55,7 +55,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( username TEXT, @@ -121,7 +121,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( username TEXT, @@ -185,7 +185,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get Mapbox isolines') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( username TEXT, @@ -249,7 +249,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get TomTom isolines') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( username TEXT, @@ -310,7 +310,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isochrones') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( username TEXT, @@ -368,7 +368,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get Mapbox isochrones') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( username TEXT, @@ -426,4 +426,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get TomTom isochrones') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index fbe192e..92882de 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- heremaps isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapzen isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -60,7 +60,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapbox isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -74,7 +74,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- tomtom isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -88,4 +88,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 731a27e..0b19f5e 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- heremaps isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapzen isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -59,7 +59,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapbox isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -72,7 +72,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- tomtom isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -85,4 +85,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index 016e70c..5099ede 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION plproxy; CREATE EXTENSION cartodb; CREATE EXTENSION cdb_geocoder; diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index a5c3652..f0a1e8e 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION plproxy; CREATE EXTENSION cartodb; CREATE EXTENSION cdb_geocoder; diff --git a/server/extension/test/sql/366_empty_table_test.sql b/server/extension/test/sql/366_empty_table_test.sql index d601c6c..ec56786 100644 --- a/server/extension/test/sql/366_empty_table_test.sql +++ b/server/extension/test/sql/366_empty_table_test.sql @@ -14,7 +14,7 @@ DO $$ service_config.user.set('soft_obs_general_limit', True) service_config.user.set('period_end_date', '20170516') -$$ LANGUAGE plpythonu; +$$ LANGUAGE @@plpythonu@@; -- Mock Observatory backend function From e08ca8f6bc0b9ead05f38f08b1bc232d6b054a8c Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 24 Feb 2020 16:13:48 +0100 Subject: [PATCH 02/21] Creates the type geomval if does not exist PG12 migration #6237 --- client/sql/05_utils.sql | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/client/sql/05_utils.sql b/client/sql/05_utils.sql index c245422..d766a3b 100644 --- a/client/sql/05_utils.sql +++ b/client/sql/05_utils.sql @@ -17,4 +17,17 @@ $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); -$f$ LANGUAGE sql IMMUTABLE; \ No newline at end of file +$f$ LANGUAGE sql IMMUTABLE; + + +-- PG12_DEPRECATED +-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster) +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN + CREATE TYPE geomval AS ( + geom geometry, + val double precision + ); + END IF; +END$$; \ No newline at end of file From 6a34f5d6cbdd77720ab759a2bb7cc385c2a41e2d Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 24 Feb 2020 16:14:29 +0100 Subject: [PATCH 03/21] Release 0.29.0 --- client/cdb_dataservices_client.control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index 395c212..ebf1daf 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.28.0' +default_version = '0.29.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client From fa2bed0df50913f8dcdeb25bcf83e74073866148 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 15:38:43 +0100 Subject: [PATCH 04/21] Add details about what the rule does --- client/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/Makefile b/client/Makefile index d292cb9..bd6c90d 100644 --- a/client/Makefile +++ b/client/Makefile @@ -107,8 +107,8 @@ ifeq ($(PG_PARALLEL), 0) done endif -# TODO -prepare_tests: +# Replacing variables defined within test files +replace_variables: mkdir -p $(TEST_DIR) mkdir -p $(TEST_DIR)/expected mkdir -p $(TEST_DIR)/sql @@ -128,4 +128,4 @@ deploy: release_remove_parallel_deploy $(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/' -install: prepare_tests deploy +install: replace_variables deploy From d5b0d869977b90abfd075910f5ddec30fb007199 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 15:41:04 +0100 Subject: [PATCH 05/21] Add variable @@plpythonu@@ Related with https://github.com/CartoDB/cartodb-platform/issues/6237 --- server/extension/Makefile | 60 ++++++++++++++++--- ...rol => cdb_dataservices_server.control.in} | 2 +- 2 files changed, 53 insertions(+), 9 deletions(-) rename server/extension/{cdb_dataservices_server.control => cdb_dataservices_server.control.in} (85%) diff --git a/server/extension/Makefile b/server/extension/Makefile index 968e51c..f160612 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -1,11 +1,10 @@ # Makefile to generate the extension out of separate sql source files. # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_dataservices_server -EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +EXTVERSION = $(shell grep default_version $(EXTENSION).control.in | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") # The new version to be generated from templates SED = sed ERB = erb -REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g' NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql AWK = awk @@ -14,7 +13,8 @@ PG_CONFIG = pg_config PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) -TEST_DIR = test/ +REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out)))) +TEST_DIR = test_out REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres' # DATA is a special variable used by postgres build infrastructure @@ -27,15 +27,29 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) SOURCES_DATA_DIR = sql/ SOURCES_DATA = $(wildcard sql/*.sql) +SOURCES_DATA_DIR_OUT = sql_out/ # postgres build stuff PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +# PG12 compatibility +PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') +PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) +PLPYTHONU := plpythonu +ifeq ($(PG_12_GE), true) +PLPYTHONU := plpython3u +endif + +REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g' +ifneq ($(NEW_VERSION),) +REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g' +endif + $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ - cat $(SOURCES_DATA_DIR)/*.sql >> $@ + cat $(SOURCES_DATA_DIR_OUT)/*.sql >> $@ ifeq ($(PG_PARALLEL), 0) # Remove PARALLEL in aggregates and functions $(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX)) @@ -44,16 +58,19 @@ ifeq ($(PG_PARALLEL), 0) mv $(TMPFILE) $@ endif +$(EXTENSION).control: $(EXTENSION).control.in Makefile + $(SED) $(REPLACEMENTS) $< > $@ + .PHONY: all -all: $(DATA) +all: replace_variables $(DATA) .PHONY: release -release: $(EXTENSION).control $(SOURCES_DATA) +release: replace_variables $(EXTENSION).control $(SOURCES_DATA) test -n "$(NEW_VERSION)" # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0 git mv *.sql old_versions - $(SED) $(REPLACEMENTS) $(EXTENSION).control + $(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control git add $(EXTENSION).control - cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql + cat $(SOURCES_DATA_DIR_OUT)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql $(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql $(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql git add $(EXTENSION)--$(NEW_VERSION).sql @@ -66,6 +83,9 @@ release: $(EXTENSION).control $(SOURCES_DATA) .PHONY: devclean devclean: rm -f $(NEW_EXTENSION_ARTIFACT) + rm -rf $(TEST_DIR) + rm -rf $(SOURCES_DATA_DIR_OUT) + rm -rf $(EXTENSION).control # If needed remove PARALLEL tags from the release files release_remove_parallel_deploy: @@ -82,6 +102,30 @@ deploy: release_remove_parallel_deploy $(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/' +# Replacing variables defined within test files +replace_variables: + # tests + mkdir -p $(TEST_DIR) + mkdir -p $(TEST_DIR)/expected + mkdir -p $(TEST_DIR)/sql + for f in $(REGRESS); do \ + cat test/sql/$${f}.sql | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + > $(TEST_DIR)/sql/$${f}.sql; \ + done + for f in $(REGRESS_EXPEC); do \ + cat test/expected/$${f}.out | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + > $(TEST_DIR)/expected/$${f}.out; \ + done + # data + mkdir -p $(SOURCES_DATA_DIR_OUT) + for f in $(notdir $(SOURCES_DATA)); do \ + cat $(SOURCES_DATA_DIR)/$${f} | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + > $(SOURCES_DATA_DIR_OUT)/$${f}; \ + done + install: deploy reinstall: install diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control.in similarity index 85% rename from server/extension/cdb_dataservices_server.control rename to server/extension/cdb_dataservices_server.control.in index 3e18400..c0d362e 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control.in @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.37.0' +default_version = '0.38.0' requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server From b2a06c6ed65f2a1d259665915387a54f93796c59 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 15:42:10 +0100 Subject: [PATCH 06/21] Create the type `geomval` if not exist Related to https://github.com/CartoDB/cartodb-platform/issues/6237 --- server/extension/sql/05_postgis3_adapter.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 server/extension/sql/05_postgis3_adapter.sql diff --git a/server/extension/sql/05_postgis3_adapter.sql b/server/extension/sql/05_postgis3_adapter.sql new file mode 100644 index 0000000..4facb46 --- /dev/null +++ b/server/extension/sql/05_postgis3_adapter.sql @@ -0,0 +1,11 @@ +-- PG12_DEPRECATED +-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster) +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN + CREATE TYPE geomval AS ( + geom geometry, + val double precision + ); + END IF; +END$$; \ No newline at end of file From 3c916bdfc9aa9f46131cf80a3d7c19a96b226b84 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 15:49:15 +0100 Subject: [PATCH 07/21] Adapt dataservices-api server to python3 Related to https://github.com/CartoDB/cartodb-platform/issues/6237 --- server/extension/sql/50_namedplaces.sql | 6 +++--- .../cartodb_services/__init__.py | 2 +- .../cartodb_services/bulk_geocoders.py | 10 +++++----- .../cartodb_services/config/__init__.py | 6 +++--- .../config/legacy_rate_limits.py | 2 +- .../cartodb_services/config/rate_limits.py | 2 +- .../cartodb_services/geocoder.py | 4 ++-- .../cartodb_services/geocodio/__init__.py | 4 ++-- .../cartodb_services/google/__init__.py | 4 ++-- .../cartodb_services/google/bulk_geocoder.py | 2 +- .../cartodb_services/google/client_factory.py | 2 +- .../cartodb_services/google/geocoder.py | 8 +++++--- .../cartodb_services/here/__init__.py | 6 +++--- .../cartodb_services/here/routing.py | 2 +- .../cartodb_services/mapbox/__init__.py | 8 ++++---- .../cartodb_services/mapzen/__init__.py | 10 +++++----- .../cartodb_services/mapzen/isolines.py | 2 +- .../cartodb_services/metrics/__init__.py | 8 ++++---- .../cartodb_services/metrics/config.py | 2 +- .../cartodb_services/metrics/log.py | 5 ++++- .../cartodb_services/metrics/quota.py | 4 ++-- .../refactor/storage/redis_config.py | 2 +- .../cartodb_services/tomtom/__init__.py | 8 ++++---- .../cartodb_services/tools/__init__.py | 18 +++++++++--------- .../cartodb_services/tools/qps.py | 2 +- test/helpers/import_helper.py | 8 ++++---- test/helpers/integration_test_helper.py | 2 +- test/run_tests.py | 8 ++++---- 28 files changed, 76 insertions(+), 71 deletions(-) diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 0e9f14e..2d3ab51 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -1,7 +1,7 @@ ---- cdb_geocode_namedplace_point(city_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) RETURNS Geometry AS $$ - import spiexceptions + from plpy import spiexceptions from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") @@ -21,7 +21,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) RETURNS Geometry AS $$ - import spiexceptions + from plpy import spiexceptions from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") @@ -41,7 +41,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ - import spiexceptions + from plpy import spiexceptions from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") diff --git a/server/lib/python/cartodb_services/cartodb_services/__init__.py b/server/lib/python/cartodb_services/cartodb_services/__init__.py index ef5caa4..68e03b5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/__init__.py @@ -34,4 +34,4 @@ def _reset(): plpy = None GD = None -from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder +from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py b/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py index ccb7d5f..f21ec9c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py +++ b/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py @@ -1,8 +1,8 @@ -from google import GoogleMapsBulkGeocoder -from here import HereMapsBulkGeocoder -from tomtom import TomTomBulkGeocoder -from mapbox import MapboxBulkGeocoder -from geocodio import GeocodioBulkGeocoder +from cartodb_services.google import GoogleMapsBulkGeocoder +from cartodb_services.here import HereMapsBulkGeocoder +from cartodb_services.tomtom import TomTomBulkGeocoder +from cartodb_services.mapbox import MapboxBulkGeocoder +from cartodb_services.geocodio import GeocodioBulkGeocoder BATCH_GEOCODER_CLASS_BY_PROVIDER = { 'google': GoogleMapsBulkGeocoder, diff --git a/server/lib/python/cartodb_services/cartodb_services/config/__init__.py b/server/lib/python/cartodb_services/cartodb_services/config/__init__.py index 6a98689..2a1c40f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/config/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/config/__init__.py @@ -1,3 +1,3 @@ -from service_configuration import ServiceConfiguration -from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter -from legacy_rate_limits import RateLimitsConfigLegacyBuilder +from .service_configuration import ServiceConfiguration +from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter +from .legacy_rate_limits import RateLimitsConfigLegacyBuilder diff --git a/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py index 11632f9..0a668a7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py +++ b/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py @@ -1,5 +1,5 @@ import json -from rate_limits import RateLimitsConfig +from cartodb_services.config.rate_limits import RateLimitsConfig class RateLimitsConfigLegacyBuilder(object): """ diff --git a/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py index 98f6a47..e09489a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py +++ b/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py @@ -1,6 +1,6 @@ import json -from service_configuration import ServiceConfiguration +from cartodb_services.config.service_configuration import ServiceConfiguration class RateLimitsConfig(object): """ diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index a2c250c..1e682a2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- -from tools import QuotaExceededException, Logger +from cartodb_services.tools import QuotaExceededException, Logger from collections import namedtuple import json @@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org except Exception as e: import sys logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname}) - metadata['processing_error'] = 'Error: {}'.format(e.message) + metadata['processing_error'] = 'Error: {}'.format(e) results.append([result[0], None, json.dumps(metadata)]) failed_count += 1 diff --git a/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py b/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py index 7b5bc90..05da24d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py @@ -1,2 +1,2 @@ -from geocoder import GeocodioGeocoder -from bulk_geocoder import GeocodioBulkGeocoder +from cartodb_services.geocodio.geocoder import GeocodioGeocoder +from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/google/__init__.py b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py index 7570e31..f249a66 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py @@ -1,2 +1,2 @@ -from geocoder import GoogleMapsGeocoder -from bulk_geocoder import GoogleMapsBulkGeocoder +from cartodb_services.google.geocoder import GoogleMapsGeocoder +from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index cde9f1c..107fb54 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -1,5 +1,5 @@ from multiprocessing import Pool -from exceptions import MalformedResult +from cartodb_services.google.exceptions import MalformedResult from cartodb_services import StreetPointBulkGeocoder from cartodb_services.geocoder import compose_address, geocoder_error_response from cartodb_services.google import GoogleMapsGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py b/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py index 9593f4f..6d936c8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py @@ -3,7 +3,7 @@ import googlemaps import base64 -from exceptions import InvalidGoogleCredentials +from cartodb_services.google.exceptions import InvalidGoogleCredentials class GoogleMapsClientFactory(): diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 30a62cf..0f96d5d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -1,9 +1,11 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- +try: + from urlparse import parse_qs +except: + from urllib.parse import parse_qs -from urlparse import parse_qs - -from exceptions import MalformedResult +from cartodb_services.google.exceptions import MalformedResult from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory diff --git a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py index 85cc596..a389e2a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py @@ -1,3 +1,3 @@ -from geocoder import HereMapsGeocoder -from bulk_geocoder import HereMapsBulkGeocoder -from routing import HereMapsRoutingIsoline +from cartodb_services.here.geocoder import HereMapsGeocoder +from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder +from cartodb_services.here.routing import HereMapsRoutingIsoline diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index fcc2e3e..ab36adf 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -1,7 +1,7 @@ import requests import json -from exceptions import WrongParams +from cartodb_services.here.exceptions import WrongParams from requests.adapters import HTTPAdapter from cartodb_services.metrics import Traceable diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py index a7afcc6..3d3416c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py @@ -1,4 +1,4 @@ -from routing import MapboxRouting, MapboxRoutingResponse -from geocoder import MapboxGeocoder -from bulk_geocoder import MapboxBulkGeocoder -from isolines import MapboxIsolines, MapboxIsochronesResponse +from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse +from cartodb_services.mapbox.geocoder import MapboxGeocoder +from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder +from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py index 831d4ad..c1d8cbc 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py @@ -1,5 +1,5 @@ -from routing import MapzenRouting, MapzenRoutingResponse -from isolines import MapzenIsolines -from geocoder import MapzenGeocoder -from matrix_client import MatrixClient -from isochrones import MapzenIsochrones +from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse +from cartodb_services.mapzen.isolines import MapzenIsolines +from cartodb_services.mapzen.geocoder import MapzenGeocoder +from cartodb_services.mapzen.matrix_client import MatrixClient +from cartodb_services.mapzen.isochrones import MapzenIsochrones diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index e6f51be..a6780f8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -135,7 +135,7 @@ class MapzenIsolines: # delete points that got None location_estimates_filtered = [] for i, c in enumerate(costs): - if c <> isorange: + if c != isorange: location_estimates_filtered.append(location_estimates[i]) return location_estimates_filtered diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index edca0ee..52cf7f3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,4 +1,4 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig -from quota import QuotaService -from user import UserMetricsService -from log import metrics, MetricsDataGatherer, Traceable +from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig +from cartodb_services.metrics.quota import QuotaService +from cartodb_services.metrics.user import UserMetricsService +from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index b762f3b..0c323f8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -45,7 +45,7 @@ class ServiceConfig(object): def _get_effective_monthly_quota(self, quota_key, default=0): quota_from_redis = self._redis_config.get(quota_key, None) - if quota_from_redis and quota_from_redis <> '': + if quota_from_redis and quota_from_redis != '': return int(quota_from_redis) else: return default diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 46b7fd2..bf66d18 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -6,7 +6,10 @@ import uuid import plpy from datetime import datetime from contextlib import contextmanager -from urlparse import urlparse +try: + from urlparse import urlparse +except: + from urllib.parse import urlparse @contextmanager diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 476318c..c532abb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -1,5 +1,5 @@ -from user import UserMetricsService -from log import MetricsDataGatherer +from cartodb_services.metrics.user import UserMetricsService +from cartodb_services.metrics.log import MetricsDataGatherer from datetime import date import re diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index 10dd21e..31e3db9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -1,5 +1,5 @@ from ..core.interfaces import ConfigBackendInterface -from null_config import NullConfigStorage +from cartodb_services.refactor.storage.null_config import NullConfigStorage class RedisConfigStorage(ConfigBackendInterface): diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py index 21d5ced..34368d8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py @@ -1,4 +1,4 @@ -from geocoder import TomTomGeocoder -from bulk_geocoder import TomTomBulkGeocoder -from routing import TomTomRouting, TomTomRoutingResponse -from isolines import TomTomIsolines, TomTomIsochronesResponse +from cartodb_services.tomtom.geocoder import TomTomGeocoder +from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder +from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse +from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index 976469a..17e9e48 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1,9 +1,9 @@ -from redis_tools import RedisConnection, RedisDBConfig -from coordinates import Coordinate -from polyline import PolyLine -from log import Logger, LoggerConfig -from rate_limiter import RateLimiter -from service_manager import ServiceManager -from legacy_service_manager import LegacyServiceManager -from exceptions import QuotaExceededException, RateLimitExceeded -from country import country_to_iso3 +from .redis_tools import RedisConnection, RedisDBConfig +from .coordinates import Coordinate +from .polyline import PolyLine +from .log import Logger, LoggerConfig +from .rate_limiter import RateLimiter +from .service_manager import ServiceManager +from .legacy_service_manager import LegacyServiceManager +from .exceptions import QuotaExceededException, RateLimitExceeded +from .country import country_to_iso3 diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py index cd52f96..17bb1ca 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py @@ -1,7 +1,7 @@ import time import random from datetime import datetime -from exceptions import TimeoutException +from cartodb_services.tools.exceptions import TimeoutException import re DEFAULT_RETRY_TIMEOUT = 60 diff --git a/test/helpers/import_helper.py b/test/helpers/import_helper.py index 59f1b23..f15913f 100644 --- a/test/helpers/import_helper.py +++ b/test/helpers/import_helper.py @@ -20,7 +20,7 @@ class ImportHelper: response.raise_for_status() response_json = json.loads(response.text) if not response_json['success']: - print "Error importing the test dataset: {0}".format(response.text) + print("Error importing the test dataset: {0}".format(response.text)) sys.exit(1) while(True): table_name = ImportHelper.get_imported_table_name( @@ -42,8 +42,8 @@ class ImportHelper: schema, username, host, import_id, api_key) import_data_response = requests.get(import_url) if import_data_response.status_code != 200: - print "Error getting the table name from " \ - "the import data: {0}".format(import_data_response.text) + print("Error getting the table name from " \ + "the import data: {0}".format(import_data_response.text)) sys.exit(1) import_data_json = json.loads(import_data_response.text) @@ -57,5 +57,5 @@ class ImportHelper: ) response = requests.get(url) if response.status_code != 200: - print "Error cleaning the test dataset: {0}".format(response.text) + print("Error cleaning the test dataset: {0}".format(response.text)) sys.exit(1) diff --git a/test/helpers/integration_test_helper.py b/test/helpers/integration_test_helper.py index df050de..02f18ee 100644 --- a/test/helpers/integration_test_helper.py +++ b/test/helpers/integration_test_helper.py @@ -48,7 +48,7 @@ class IntegrationTestHelper: def execute_query_raw(cls, sql_api_url, query): requests.packages.urllib3.disable_warnings() query_url = "{0}?q={1}".format(sql_api_url, query) - print "Executing query: {0}".format(query_url) + print("Executing query: {0}".format(query_url)) query_response = requests.get(query_url) if query_response.status_code != 200: raise Exception(json.loads(query_response.text)['error']) diff --git a/test/run_tests.py b/test/run_tests.py index 7308010..849822f 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -36,7 +36,7 @@ def main(): set_environment_variables(username, api_key, table_name, host, schema) execute_tests() except Exception as e: - print e.message + print(e) sys.exit(1) finally: clean_environment_variables() @@ -45,11 +45,11 @@ def main(): def usage(): - print """Usage: run_tests.py [options] username api_key + print("""Usage: run_tests.py [options] username api_key Options: -h: Show this help --host: take that host as base (by default is cartodb.com) - --schema: define the url schema [http/https] (by default https)""" + --schema: define the url schema [http/https] (by default https)""") def execute_tests(): @@ -59,7 +59,7 @@ def execute_tests(): stderr=subprocess.PIPE ) out, err = process.communicate() - print err + print(err) regexp = re.compile(r'FAILED \(.*\)') if regexp.search(err) is not None: sys.exit(1) From 53a0e7abf3288435c7eca4ec6434725401ff46ca Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 17:32:08 +0100 Subject: [PATCH 08/21] Show errors if tests failed --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09bf19f..e6cc862 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,9 +44,9 @@ jobs: timeout-minutes: 5 - name: Run server tests - run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck" + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || cat /dataservices-api/server/extension/test_out/regression.diffs" timeout-minutes: 5 - name: Run client tests - run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck" + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs" timeout-minutes: 5 From 8a3f28c45bc23a2a2fb35ecb297e78f4a3a12fa1 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 17:32:59 +0100 Subject: [PATCH 09/21] Rename output test directory --- client/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/Makefile b/client/Makefile index bd6c90d..3e517e6 100644 --- a/client/Makefile +++ b/client/Makefile @@ -35,7 +35,7 @@ SOURCES_DATA_DIR = sql/ REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out)))) -TEST_DIR = test_re +TEST_DIR = test_out REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres' # DATA is a special variable used by postgres build infrastructure From 35f098686c9ec83ff848dc506316fcfdc1512671 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 17:39:35 +0100 Subject: [PATCH 10/21] Force non-zero exit if tests failed --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e6cc862..84185b4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,9 +44,9 @@ jobs: timeout-minutes: 5 - name: Run server tests - run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || cat /dataservices-api/server/extension/test_out/regression.diffs" + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test_out/regression.diffs && false)" timeout-minutes: 5 - name: Run client tests - run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs" + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs && false)" timeout-minutes: 5 From 7aefba181c841b71a5c4d3b34d09b538d9535556 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Wed, 26 Feb 2020 18:52:26 +0100 Subject: [PATCH 11/21] Install packages in python3 using pip3 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 84185b4..49bad27 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,7 @@ jobs: run: docker-compose -f docker-compose.yaml up -d - name: Install required python libs - run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services" + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services" - name: Run python library tests run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && MAPBOX_API_KEY=$MAPBOX_API_KEY TOMTOM_API_KEY=$TOMTOM_API_KEY GEOCODIO_API_KEY=$GEOCODIO_API_KEY nosetests lib/python/cartodb_services/test" From 5370ea23c78e9339c943dde23923bd9e788d4ff0 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Thu, 27 Feb 2020 10:14:42 +0100 Subject: [PATCH 12/21] Conditional jobs dependint on PG version --- .github/workflows/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 49bad27..9b53b35 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,8 +32,13 @@ jobs: - name: Start docker-compose services run: docker-compose -f docker-compose.yaml up -d - - name: Install required python libs + - name: Install required python3 libs run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services" + if: ${{ PG_VERSION == 12 }} + + - name: Install required python libs + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services" + if: ${{ PG_VERSION == 10 }} - name: Run python library tests run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && MAPBOX_API_KEY=$MAPBOX_API_KEY TOMTOM_API_KEY=$TOMTOM_API_KEY GEOCODIO_API_KEY=$GEOCODIO_API_KEY nosetests lib/python/cartodb_services/test" From 4a880805ccdfd115bcccca411540b5b1553f66d3 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Thu, 27 Feb 2020 10:19:13 +0100 Subject: [PATCH 13/21] Fix conditional CI --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b53b35..f2614c8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,11 +34,11 @@ jobs: - name: Install required python3 libs run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services" - if: ${{ PG_VERSION == 12 }} + if: env.PG_VERSION == 12 - name: Install required python libs run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services" - if: ${{ PG_VERSION == 10 }} + if: env.PG_VERSION == 10 - name: Run python library tests run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && MAPBOX_API_KEY=$MAPBOX_API_KEY TOMTOM_API_KEY=$TOMTOM_API_KEY GEOCODIO_API_KEY=$GEOCODIO_API_KEY nosetests lib/python/cartodb_services/test" From 0164b07b921c6a853860f96364d539d151dfa106 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Thu, 27 Feb 2020 10:46:06 +0100 Subject: [PATCH 14/21] Install always python lib in python2 for unit tests --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2614c8..16003ff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,6 @@ jobs: - name: Install required python libs run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services" - if: env.PG_VERSION == 10 - name: Run python library tests run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && MAPBOX_API_KEY=$MAPBOX_API_KEY TOMTOM_API_KEY=$TOMTOM_API_KEY GEOCODIO_API_KEY=$GEOCODIO_API_KEY nosetests lib/python/cartodb_services/test" From 880f756eaec3952a9ed9f3ee2194288af235ba91 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Thu, 27 Feb 2020 16:06:30 +0100 Subject: [PATCH 15/21] Add the extension schema to the SEARCH_PATH --- .../extension/test/expected/100_data_observatory_test.out | 1 + server/extension/test/sql/100_data_observatory_test.sql | 7 +++++++ server/extension/test/sql/366_empty_table_test.sql | 7 ++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out index 1a5fd7a..b3d7d58 100644 --- a/server/extension/test/expected/100_data_observatory_test.out +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -1,3 +1,4 @@ +\set ECHO none SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/100_data_observatory_test.sql b/server/extension/test/sql/100_data_observatory_test.sql index 200024b..2f9c374 100644 --- a/server/extension/test/sql/100_data_observatory_test.sql +++ b/server/extension/test/sql/100_data_observatory_test.sql @@ -1,3 +1,10 @@ +\set ECHO none +-- add the schema cdb_dataservices_server to the SEARCH_PATH +DO $$ BEGIN + PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false); +END $$; +\set ECHO all + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/366_empty_table_test.sql b/server/extension/test/sql/366_empty_table_test.sql index ec56786..7b24a74 100644 --- a/server/extension/test/sql/366_empty_table_test.sql +++ b/server/extension/test/sql/366_empty_table_test.sql @@ -2,6 +2,11 @@ \set VERBOSITY verbose SET client_min_messages TO error; +-- add the schema cdb_dataservices_server to the SEARCH_PATH +DO $$ BEGIN + PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false); +END $$; + -- Set configuration for a user 'foo' DO $$ import json @@ -37,7 +42,7 @@ GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api; SELECT * FROM cdb_dataservices_server.OBS_GetData( 'foo', NULL, - '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::public._geomval, + '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::_geomval, '[{"id": 1, "score": 52.7515548093083898758340051256007949661290516400338, "geom_id": "us.census.tiger.census_tract", "denom_id": "us.census.acs.B01003001", "numer_id": "us.census.acs.B03002003", "geom_name": "US Census Tracts", "geom_type": "Geometry", "num_geoms": 2.86483076549783307739486952736, "denom_name": "Total Population", "denom_type": "Numeric", "numer_name": "White Population", "numer_type": "Numeric", "score_rank": 1, "target_area": 0.000307374806576033, "geom_colname": "the_geom", "score_rownum": 1, "target_geoms": null, "denom_colname": "total_pop", "denom_reltype": ' '"denominator", "geom_timespan": "2015", "normalization": "prenormalized", "numer_colname": "white_pop", "timespan_rank": 1, "geom_tablename": "obs_87a814e485deabe3b12545a537f693d16ca702c2", "max_score_rank": null, "numer_timespan": "2010 - 2014", "suggested_name": "white_pop_2010_2014", "denom_aggregate": "sum", "denom_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "numer_aggregate": "sum", "numer_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "timespan_rownum": 1, "geom_description": "Census tracts are small, relatively permanent statistical subdivisions of a county or equivalent entity that are updated by local participants prior to each decennial census as part of the Census Bureau’s Participant Statistical Areas Program. The Census Bureau delineates census tracts in situations where no local participant existed or where state, local, or tribal governments' 'declined to participate. The primary purpose of census tracts is to provide a stable set of geographic units for the presentation of statistical data.\r\n\r\nCensus tracts generally have a population size between 1,200 and 8,000 people, with an optimum size of 4,000 people. A census tract usually covers a contiguous area; however, the spatial size of census tracts varies widely depending on the density of settlement. Census tract boundaries are delineated with the intention of being maintained over a long time so that statistical comparisons can be made from census to census. Census tracts occasionally are split due to population growth or merged as a result of substantial population decline.\r\n\r\nCensus tract boundaries generally follow visible and identifiable features. They may follow nonvisible legal boundaries, such as minor civil division (MCD) or incorporated place boundaries' From e97d350e5acb109732aba42748fb220e5dc19a9f Mon Sep 17 00:00:00 2001 From: manmorjim Date: Thu, 27 Feb 2020 17:19:15 +0100 Subject: [PATCH 16/21] Decode redis responses by default In python3 redis responses are encoded by default. --- .../cartodb_services/refactor/storage/redis_connection.py | 2 +- .../cartodb_services/cartodb_services/tools/redis_tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py index a125af3..4350358 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py @@ -18,5 +18,5 @@ class RedisConnectionBuilder(): else: conn = StrictRedis(host=self._config.host, port=self._config.port, db=self._config.db, retry_on_timeout=True, - socket_timeout=self._config.timeout) + socket_timeout=self._config.timeout, decode_responses=True) return conn diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index e8b5800..48c8531 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -23,7 +23,7 @@ class RedisConnection: else: conn = StrictRedis(host=self._config.host, port=self._config.port, db=self._config.db, retry_on_timeout=True, - socket_timeout=self._config.timeout) + socket_timeout=self._config.timeout, decode_responses=True) return conn From 944bc8c3ffaf8b58c99be989beb2d7b377af2c47 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 2 Mar 2020 09:09:10 +0100 Subject: [PATCH 17/21] Create the type `geomval` in the extension schema Having it as public.geomval means that we would block ourselves from installing `postgis_raster` if this extension is installed. --- client/sql/05_utils.sql | 2 +- server/extension/sql/05_postgis3_adapter.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/sql/05_utils.sql b/client/sql/05_utils.sql index d766a3b..582a5d0 100644 --- a/client/sql/05_utils.sql +++ b/client/sql/05_utils.sql @@ -25,7 +25,7 @@ $f$ LANGUAGE sql IMMUTABLE; DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN - CREATE TYPE geomval AS ( + CREATE TYPE cdb_dataservices_client.geomval AS ( geom geometry, val double precision ); diff --git a/server/extension/sql/05_postgis3_adapter.sql b/server/extension/sql/05_postgis3_adapter.sql index 4facb46..addad2e 100644 --- a/server/extension/sql/05_postgis3_adapter.sql +++ b/server/extension/sql/05_postgis3_adapter.sql @@ -3,7 +3,7 @@ DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN - CREATE TYPE geomval AS ( + CREATE TYPE cdb_dataservices_server.geomval AS ( geom geometry, val double precision ); From e3e037563b548646913f8eb242725188ed3ef298 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 2 Mar 2020 15:21:32 +0100 Subject: [PATCH 18/21] Deconding the tests output --- test/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_tests.py b/test/run_tests.py index 849822f..a505750 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -59,7 +59,7 @@ def execute_tests(): stderr=subprocess.PIPE ) out, err = process.communicate() - print(err) + print(err.decode('utf-8')) regexp = re.compile(r'FAILED \(.*\)') if regexp.search(err) is not None: sys.exit(1) From 5bb0e674998c53d9e339b09ea05a9e4645bb0fca Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 2 Mar 2020 15:24:56 +0100 Subject: [PATCH 19/21] Add method parameter to requests It allows using POST for large URI to prevent errors `414 Request-URI Too Large` --- test/helpers/integration_test_helper.py | 8 ++++++-- test/integration/test_street_functions.py | 16 +++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/test/helpers/integration_test_helper.py b/test/helpers/integration_test_helper.py index 02f18ee..76ffcd8 100644 --- a/test/helpers/integration_test_helper.py +++ b/test/helpers/integration_test_helper.py @@ -45,11 +45,15 @@ class IntegrationTestHelper: } @classmethod - def execute_query_raw(cls, sql_api_url, query): + def execute_query_raw(cls, sql_api_url, query, method='GET'): requests.packages.urllib3.disable_warnings() + if method.upper() == 'GET': query_url = "{0}?q={1}".format(sql_api_url, query) - print("Executing query: {0}".format(query_url)) query_response = requests.get(query_url) + print("Executing query GET: {0}".format(query_url)) + else: + query_response = requests.post(sql_api_url, data={"q": query}) + print("Executing query POST: {0}".format(sql_api_url)) if query_response.status_code != 200: raise Exception(json.loads(query_response.text)['error']) return json.loads(query_response.text) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index fd1b983..8ddbce6 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -335,7 +335,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "{}" \ "]''::jsonb) as (cartodb_id integer, address text)', " \ "'address', null, null, null, {})".format(','.join(streets), batch_size) - response = self._run_authenticated(query) + response = self._run_authenticated(query, method='POST') assert_equal(n, len(response['rows'])) for row in response['rows']: assert_not_equal(row['st_x'], None) @@ -413,12 +413,14 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): for r, e in zip(response['rows'], expected): self.assert_metadata(r['metadata'], e) - def _run_authenticated(self, query): - authenticated_query = "{}&api_key={}".format(query, - self.env_variables[ - 'api_key']) - return IntegrationTestHelper.execute_query_raw(self.sql_api_url, - authenticated_query) + def _run_authenticated(self, query, method='GET'): + api_key = self.env_variables['api_key'] + url = self.sql_api_url + auth_query = "{}&api_key={}".format(query, api_key) + if method.upper() != 'GET': + auth_query = query + url = "{}?api_key={}".format(self.sql_api_url, api_key) + return IntegrationTestHelper.execute_query_raw(url, auth_query, method) @staticmethod def _x_y_by_cartodb_id(response): From 2664095e21841b4577f115db61988a90a9d1ed60 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 2 Mar 2020 15:26:59 +0100 Subject: [PATCH 20/21] Update tolerance and fixture values --- test/helpers/integration_test_helper.py | 8 ++++---- test/integration/test_street_functions.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/helpers/integration_test_helper.py b/test/helpers/integration_test_helper.py index 76ffcd8..4d07382 100644 --- a/test/helpers/integration_test_helper.py +++ b/test/helpers/integration_test_helper.py @@ -10,7 +10,7 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) -def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005): +def assert_close_enough(xy_a, xy_b, rel_tol=0.001, abs_tol=0.0005): """ Asserts that the given points are "close enough", in a square. :param xy_a: Array of 2 elements, X and Y. @@ -23,7 +23,7 @@ def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005): assert_true(isclose(xy_a[i], xy_b[i], rel_tol, abs_tol), "Coord {} error: {} and {} are not closer than {}, {}".format( i, xy_a[i], xy_b[i], rel_tol, abs_tol - )) + )) class IntegrationTestHelper: @@ -48,8 +48,8 @@ class IntegrationTestHelper: def execute_query_raw(cls, sql_api_url, query, method='GET'): requests.packages.urllib3.disable_warnings() if method.upper() == 'GET': - query_url = "{0}?q={1}".format(sql_api_url, query) - query_response = requests.get(query_url) + query_url = "{0}?q={1}".format(sql_api_url, query) + query_response = requests.get(query_url) print("Executing query GET: {0}".format(query_url)) else: query_response = requests.post(sql_api_url, data={"q": query}) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 8ddbce6..9d8a7f9 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -85,9 +85,9 @@ class TestStreetFunctionsSetUp(TestCase): TOMTOM_METADATAS = { 'Plaza España, Barcelona': - {'relevance': 0.85, 'precision': 'precise', 'match_types': ['street']}, + {'relevance': 0.72, 'precision': 'precise', 'match_types': ['street']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.45, 'precision': 'interpolated', 'match_types': ['street']} + {'relevance': 0.74, 'precision': 'precise', 'match_types': ['street']} } MAPBOX_METADATAS = { From e99d88500a16c8f242f2037b632b4b91797fdf86 Mon Sep 17 00:00:00 2001 From: manmorjim Date: Mon, 2 Mar 2020 15:27:32 +0100 Subject: [PATCH 21/21] Use authenticated query for geocoding --- test/integration/test_street_functions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 9d8a7f9..2350ff5 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -151,10 +151,9 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): query = "SELECT cdb_dataservices_client.cdb_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1".format(table) try: - IntegrationTestHelper.execute_query(self.sql_api_url, query) + self._run_authenticated(query)['rows'][0] except Exception as e: - assert_equal(e.message[0], - "permission denied for relation {}".format(table)) + assert_equal(e, "permission denied for relation {}".format(table)) def test_component_aggregation(self): query = "select st_x(the_geom), st_y(the_geom) from (" \