diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09bf19f..16003ff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,6 +32,10 @@ jobs: - name: Start docker-compose services run: docker-compose -f docker-compose.yaml up -d + - name: Install required python3 libs + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services" + if: env.PG_VERSION == 12 + - name: Install required python libs run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services" @@ -44,9 +48,9 @@ jobs: timeout-minutes: 5 - name: Run server tests - run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck" + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test_out/regression.diffs && false)" timeout-minutes: 5 - name: Run client tests - run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck" + run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs && false)" timeout-minutes: 5 diff --git a/client/Makefile b/client/Makefile index 50c7102..3e517e6 100644 --- a/client/Makefile +++ b/client/Makefile @@ -14,6 +14,14 @@ AWK = awk PG_CONFIG = pg_config PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) +# PG12 compatibility +PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') +PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) +PLPYTHONU := plpythonu +ifeq ($(PG_12_GE), true) +PLPYTHONU := plpython3u +endif + # OLD_VERSIONS = $(wildcard old_versions/*.sql) # DATA = $(NEW_EXTENSION_ARTIFACT) \ # $(OLD_VERSIONS) \ @@ -25,9 +33,9 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) SOURCES_DATA_DIR = sql/ - REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) -TEST_DIR = test/ +REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out)))) +TEST_DIR = test_out REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres' # DATA is a special variable used by postgres build infrastructure @@ -54,7 +62,8 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES) $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ - cat $(SOURCES_DATA_DIR)/*.sql >> $@ + cat $(SOURCES_DATA_DIR)/*.sql | \ + $(SED) -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@ ifeq ($(PG_PARALLEL), 0) # Remove PARALLEL in aggregates and functions $(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX)) @@ -86,6 +95,7 @@ release: $(EXTENSION).control $(SOURCES_DATA) devclean: rm -f $(NEW_EXTENSION_ARTIFACT) rm -f $(GENERATED_SQL_FILES) + rm -rf $(TEST_DIR) # If needed remove PARALLEL tags from the release files release_remove_parallel_deploy: @@ -97,9 +107,25 @@ ifeq ($(PG_PARALLEL), 0) done endif +# Replacing variables defined within test files +replace_variables: + mkdir -p $(TEST_DIR) + mkdir -p $(TEST_DIR)/expected + mkdir -p $(TEST_DIR)/sql + for f in $(REGRESS); do \ + cat test/sql/$${f}.sql | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + >> $(TEST_DIR)/sql/$${f}.sql; \ + done + for f in $(REGRESS_EXPEC); do \ + cat test/expected/$${f}.out | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + >> $(TEST_DIR)/expected/$${f}.out; \ + done + # Install the current release into the PostgreSQL extensions directory deploy: release_remove_parallel_deploy $(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/' -install: deploy +install: replace_variables deploy diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index 395c212..ebf1daf 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.28.0' +default_version = '0.29.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/client/sql/05_utils.sql b/client/sql/05_utils.sql index c245422..582a5d0 100644 --- a/client/sql/05_utils.sql +++ b/client/sql/05_utils.sql @@ -17,4 +17,17 @@ $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); -$f$ LANGUAGE sql IMMUTABLE; \ No newline at end of file +$f$ LANGUAGE sql IMMUTABLE; + + +-- PG12_DEPRECATED +-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster) +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN + CREATE TYPE cdb_dataservices_client.geomval AS ( + geom geometry, + val double precision + ); + END IF; +END$$; \ No newline at end of file diff --git a/client/sql/20_table_augmentation.sql b/client/sql/20_table_augmentation.sql index 3ab46a9..7dd48e8 100644 --- a/client/sql/20_table_augmentation.sql +++ b/client/sql/20_table_augmentation.sql @@ -138,7 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeas ) return True -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( username text, @@ -231,7 +231,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMea fdw_server=plpy.quote_literal(server_name))) return True -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable( username text, diff --git a/client/test/expected/00_installation_test.out b/client/test/expected/00_installation_test.out index 650b141..fb6acde 100644 --- a/client/test/expected/00_installation_test.out +++ b/client/test/expected/00_installation_test.out @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION cartodb; CREATE EXTENSION plproxy; -- Install the extension diff --git a/client/test/sql/00_installation_test.sql b/client/test/sql/00_installation_test.sql index 49eab80..89afd26 100644 --- a/client/test/sql/00_installation_test.sql +++ b/client/test/sql/00_installation_test.sql @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION cartodb; CREATE EXTENSION plproxy; diff --git a/server/extension/Makefile b/server/extension/Makefile index 968e51c..f160612 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -1,11 +1,10 @@ # Makefile to generate the extension out of separate sql source files. # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_dataservices_server -EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +EXTVERSION = $(shell grep default_version $(EXTENSION).control.in | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") # The new version to be generated from templates SED = sed ERB = erb -REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g' NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql AWK = awk @@ -14,7 +13,8 @@ PG_CONFIG = pg_config PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) -TEST_DIR = test/ +REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out)))) +TEST_DIR = test_out REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres' # DATA is a special variable used by postgres build infrastructure @@ -27,15 +27,29 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) SOURCES_DATA_DIR = sql/ SOURCES_DATA = $(wildcard sql/*.sql) +SOURCES_DATA_DIR_OUT = sql_out/ # postgres build stuff PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +# PG12 compatibility +PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') +PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) +PLPYTHONU := plpythonu +ifeq ($(PG_12_GE), true) +PLPYTHONU := plpython3u +endif + +REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g' +ifneq ($(NEW_VERSION),) +REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g' +endif + $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ - cat $(SOURCES_DATA_DIR)/*.sql >> $@ + cat $(SOURCES_DATA_DIR_OUT)/*.sql >> $@ ifeq ($(PG_PARALLEL), 0) # Remove PARALLEL in aggregates and functions $(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX)) @@ -44,16 +58,19 @@ ifeq ($(PG_PARALLEL), 0) mv $(TMPFILE) $@ endif +$(EXTENSION).control: $(EXTENSION).control.in Makefile + $(SED) $(REPLACEMENTS) $< > $@ + .PHONY: all -all: $(DATA) +all: replace_variables $(DATA) .PHONY: release -release: $(EXTENSION).control $(SOURCES_DATA) +release: replace_variables $(EXTENSION).control $(SOURCES_DATA) test -n "$(NEW_VERSION)" # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0 git mv *.sql old_versions - $(SED) $(REPLACEMENTS) $(EXTENSION).control + $(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control git add $(EXTENSION).control - cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql + cat $(SOURCES_DATA_DIR_OUT)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql $(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql $(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql git add $(EXTENSION)--$(NEW_VERSION).sql @@ -66,6 +83,9 @@ release: $(EXTENSION).control $(SOURCES_DATA) .PHONY: devclean devclean: rm -f $(NEW_EXTENSION_ARTIFACT) + rm -rf $(TEST_DIR) + rm -rf $(SOURCES_DATA_DIR_OUT) + rm -rf $(EXTENSION).control # If needed remove PARALLEL tags from the release files release_remove_parallel_deploy: @@ -82,6 +102,30 @@ deploy: release_remove_parallel_deploy $(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/' +# Replacing variables defined within test files +replace_variables: + # tests + mkdir -p $(TEST_DIR) + mkdir -p $(TEST_DIR)/expected + mkdir -p $(TEST_DIR)/sql + for f in $(REGRESS); do \ + cat test/sql/$${f}.sql | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + > $(TEST_DIR)/sql/$${f}.sql; \ + done + for f in $(REGRESS_EXPEC); do \ + cat test/expected/$${f}.out | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + > $(TEST_DIR)/expected/$${f}.out; \ + done + # data + mkdir -p $(SOURCES_DATA_DIR_OUT) + for f in $(notdir $(SOURCES_DATA)); do \ + cat $(SOURCES_DATA_DIR)/$${f} | \ + sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + > $(SOURCES_DATA_DIR_OUT)/$${f}; \ + done + install: deploy reinstall: install diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control.in similarity index 53% rename from server/extension/cdb_dataservices_server.control rename to server/extension/cdb_dataservices_server.control.in index 9788bda..c0d362e 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control.in @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.37.0' -requires = 'plpythonu, plproxy, postgis, cdb_geocoder' +default_version = '0.38.0' +requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/sql/05_postgis3_adapter.sql b/server/extension/sql/05_postgis3_adapter.sql new file mode 100644 index 0000000..addad2e --- /dev/null +++ b/server/extension/sql/05_postgis3_adapter.sql @@ -0,0 +1,11 @@ +-- PG12_DEPRECATED +-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster) +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN + CREATE TYPE cdb_dataservices_server.geomval AS ( + geom geometry, + val double precision + ); + END IF; +END$$; \ No newline at end of file diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index e5f18ad..eeafba5 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -69,7 +69,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate Mapbox routing') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( username TEXT, @@ -137,7 +137,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate TomTom routing') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( username TEXT, @@ -198,4 +198,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate mapzen routing') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 537feb7..e5d4ae6 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -37,7 +37,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( @@ -76,4 +76,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/10_redis_helper.sql b/server/extension/sql/10_redis_helper.sql index 1ba0ca4..2b39b03 100644 --- a/server/extension/sql/10_redis_helper.sql +++ b/server/extension/sql/10_redis_helper.sql @@ -15,4 +15,4 @@ RETURNS boolean AS $$ 'redis_metrics_connection': redis_metrics_connection, } return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index 95f8f20..738262f 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -14,7 +14,7 @@ RETURNS text AS $$ user_obs_config = GD["user_obs_config_{0}".format(username)] return user_obs_config.connection_str -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( username TEXT, @@ -68,7 +68,7 @@ RETURNS json AS $$ raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( username TEXT, @@ -125,7 +125,7 @@ RETURNS SETOF JSON AS $$ raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( username TEXT, @@ -177,7 +177,7 @@ RETURNS json AS $$ raise Exception('Error trying to obs_get_segment_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( username TEXT, @@ -232,7 +232,7 @@ RETURNS SETOF JSON AS $$ raise Exception('Error trying to OBS_GetSegmentSnapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( username TEXT, @@ -289,7 +289,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetMeasure') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( username TEXT, @@ -344,7 +344,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetCategory') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( username TEXT, @@ -401,7 +401,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetUSCensusMeasure') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( username TEXT, @@ -456,7 +456,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetUSCensusCategory') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( username TEXT, @@ -511,7 +511,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetPopulation') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( username TEXT, @@ -566,7 +566,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( username TEXT, @@ -627,7 +627,7 @@ RETURNS TABLE ( raise Exception('Error trying to OBS_GetData') finally: quota_service.increment_total_service_use(len(geomvals)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( username TEXT, @@ -688,7 +688,7 @@ RETURNS TABLE ( raise Exception('Error trying to OBS_GetData') finally: quota_service.increment_total_service_use(len(geomrefs)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( username TEXT, @@ -736,7 +736,7 @@ RETURNS JSON AS $$ import sys logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to OBS_GetMeta') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( username TEXT, @@ -782,4 +782,4 @@ RETURNS TABLE(valid boolean, errors text[]) AS $$ import sys logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to OBS_MetadataValidation') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index fcdc743..faab9a7 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -55,7 +55,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) raise Exception('Error trying to OBS_Search') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( username TEXT, @@ -113,4 +113,4 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index c0b6290..9524378 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -49,7 +49,7 @@ RETURNS geometry(Geometry, 4326) AS $$ raise Exception('Error trying to OBS_GetBoundary') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( username TEXT, @@ -102,7 +102,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetBoundaryId') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( username TEXT, @@ -155,7 +155,7 @@ RETURNS geometry(Geometry, 4326) AS $$ raise Exception('Error trying to OBS_GetBoundaryById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( username TEXT, @@ -215,7 +215,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetBoundariesByGeometry') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( username TEXT, @@ -277,7 +277,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( username TEXT, @@ -337,7 +337,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetPointsByGeometry') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( username TEXT, @@ -399,4 +399,4 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetPointsByPointAndRadius') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/125_data_observatory_table_augment.sql b/server/extension/sql/125_data_observatory_table_augment.sql index e64121e..6e1bad3 100644 --- a/server/extension/sql/125_data_observatory_table_augment.sql +++ b/server/extension/sql/125_data_observatory_table_augment.sql @@ -8,7 +8,7 @@ RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) )[0] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 6e4d884..49f7735 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -8,7 +8,7 @@ RETURNS boolean AS $$ logger_config = LoggerConfig(plpy) GD[cache_key] = logger_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; -- This is done in order to avoid an undesired depedency on cartodb extension CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) @@ -46,7 +46,7 @@ RETURNS boolean AS $$ geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) GD[cache_key] = geocoder_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ @@ -60,7 +60,7 @@ RETURNS boolean AS $$ geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) GD[cache_key] = geocoder_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) RETURNS boolean AS $$ @@ -74,7 +74,7 @@ RETURNS boolean AS $$ isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = isolines_routing_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) RETURNS boolean AS $$ @@ -88,7 +88,7 @@ RETURNS boolean AS $$ routing_config = RoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = routing_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) RETURNS boolean AS $$ @@ -102,4 +102,4 @@ RETURNS boolean AS $$ obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) GD[cache_key] = obs_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 973d1ce..23ee403 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -106,7 +106,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ ret += [[service, monthly_quota, used_quota, soft_limit, provider]] return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( @@ -136,7 +136,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 76c01be..00674f3 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ else: raise Exception('Requested geocoder is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) @@ -52,7 +52,7 @@ RETURNS Geometry AS $$ else: raise Exception('Here geocoder is not available for your account.') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -67,7 +67,7 @@ RETURNS Geometry AS $$ else: raise Exception('Google geocoder is not available for your account.') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -80,7 +80,7 @@ RETURNS Geometry AS $$ mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -93,7 +93,7 @@ RETURNS Geometry AS $$ mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -106,7 +106,7 @@ RETURNS Geometry AS $$ tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -119,7 +119,7 @@ RETURNS Geometry AS $$ geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -152,7 +152,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using here maps') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -184,7 +184,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using google maps') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -225,7 +225,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using mapzen') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -271,7 +271,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using Mapbox') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -317,7 +317,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using TomTom') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -363,4 +363,4 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using Geocodio') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/210_rates.sql b/server/extension/sql/210_rates.sql index 6fa97d7..f83f0da 100644 --- a/server/extension/sql/210_rates.sql +++ b/server/extension/sql/210_rates.sql @@ -16,7 +16,7 @@ RETURNS JSON AS $$ return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) else: return None -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( username TEXT, @@ -40,7 +40,7 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_user_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( username TEXT, @@ -64,7 +64,7 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_org_rate_limits(config) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( username TEXT, @@ -88,4 +88,4 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_server_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 8e0b5ea..e41e67e 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -40,7 +40,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) return plpy.execute(plan, [username, orgname, searches]) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -51,7 +51,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = LegacyServiceManager('geocoder', username, orgname, GD) geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -62,7 +62,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = LegacyServiceManager('geocoder', username, orgname, GD) geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -79,7 +79,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -96,7 +96,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ @@ -113,4 +113,4 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD) geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index c5d8c8e..dead116 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -35,7 +35,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin0 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index c10e3b2..75ac7dd 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) @@ -74,7 +74,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 932fc78..2d3ab51 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -1,7 +1,7 @@ ---- cdb_geocode_namedplace_point(city_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) RETURNS Geometry AS $$ - import spiexceptions + from plpy import spiexceptions from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") @@ -16,12 +16,12 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) RETURNS Geometry AS $$ - import spiexceptions + from plpy import spiexceptions from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") @@ -36,12 +36,12 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ - import spiexceptions + from plpy import spiexceptions from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") @@ -56,7 +56,7 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ @@ -101,7 +101,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode namedplace point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index d82f42e..fcd0e53 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -35,7 +35,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) RETURNS Geometry AS $$ @@ -77,7 +77,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ @@ -119,7 +119,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) RETURNS Geometry AS $$ @@ -162,7 +162,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 0f41c5a..8fa025f 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -35,7 +35,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index a259676..af65951 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -55,7 +55,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( username TEXT, @@ -121,7 +121,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( username TEXT, @@ -185,7 +185,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get Mapbox isolines') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( username TEXT, @@ -249,7 +249,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get TomTom isolines') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( username TEXT, @@ -310,7 +310,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isochrones') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( username TEXT, @@ -368,7 +368,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get Mapbox isochrones') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( username TEXT, @@ -426,4 +426,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get TomTom isochrones') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index fbe192e..92882de 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- heremaps isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapzen isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -60,7 +60,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapbox isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -74,7 +74,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- tomtom isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -88,4 +88,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 731a27e..0b19f5e 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- heremaps isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapzen isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -59,7 +59,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- mapbox isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -72,7 +72,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; -- tomtom isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -85,4 +85,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index 016e70c..5099ede 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION plproxy; CREATE EXTENSION cartodb; CREATE EXTENSION cdb_geocoder; diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out index 1a5fd7a..b3d7d58 100644 --- a/server/extension/test/expected/100_data_observatory_test.out +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -1,3 +1,4 @@ +\set ECHO none SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index a5c3652..f0a1e8e 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE EXTENSION plproxy; CREATE EXTENSION cartodb; CREATE EXTENSION cdb_geocoder; diff --git a/server/extension/test/sql/100_data_observatory_test.sql b/server/extension/test/sql/100_data_observatory_test.sql index 200024b..2f9c374 100644 --- a/server/extension/test/sql/100_data_observatory_test.sql +++ b/server/extension/test/sql/100_data_observatory_test.sql @@ -1,3 +1,10 @@ +\set ECHO none +-- add the schema cdb_dataservices_server to the SEARCH_PATH +DO $$ BEGIN + PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false); +END $$; +\set ECHO all + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/366_empty_table_test.sql b/server/extension/test/sql/366_empty_table_test.sql index d601c6c..7b24a74 100644 --- a/server/extension/test/sql/366_empty_table_test.sql +++ b/server/extension/test/sql/366_empty_table_test.sql @@ -2,6 +2,11 @@ \set VERBOSITY verbose SET client_min_messages TO error; +-- add the schema cdb_dataservices_server to the SEARCH_PATH +DO $$ BEGIN + PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false); +END $$; + -- Set configuration for a user 'foo' DO $$ import json @@ -14,7 +19,7 @@ DO $$ service_config.user.set('soft_obs_general_limit', True) service_config.user.set('period_end_date', '20170516') -$$ LANGUAGE plpythonu; +$$ LANGUAGE @@plpythonu@@; -- Mock Observatory backend function @@ -37,7 +42,7 @@ GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api; SELECT * FROM cdb_dataservices_server.OBS_GetData( 'foo', NULL, - '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::public._geomval, + '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::_geomval, '[{"id": 1, "score": 52.7515548093083898758340051256007949661290516400338, "geom_id": "us.census.tiger.census_tract", "denom_id": "us.census.acs.B01003001", "numer_id": "us.census.acs.B03002003", "geom_name": "US Census Tracts", "geom_type": "Geometry", "num_geoms": 2.86483076549783307739486952736, "denom_name": "Total Population", "denom_type": "Numeric", "numer_name": "White Population", "numer_type": "Numeric", "score_rank": 1, "target_area": 0.000307374806576033, "geom_colname": "the_geom", "score_rownum": 1, "target_geoms": null, "denom_colname": "total_pop", "denom_reltype": ' '"denominator", "geom_timespan": "2015", "normalization": "prenormalized", "numer_colname": "white_pop", "timespan_rank": 1, "geom_tablename": "obs_87a814e485deabe3b12545a537f693d16ca702c2", "max_score_rank": null, "numer_timespan": "2010 - 2014", "suggested_name": "white_pop_2010_2014", "denom_aggregate": "sum", "denom_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "numer_aggregate": "sum", "numer_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "timespan_rownum": 1, "geom_description": "Census tracts are small, relatively permanent statistical subdivisions of a county or equivalent entity that are updated by local participants prior to each decennial census as part of the Census Bureau’s Participant Statistical Areas Program. The Census Bureau delineates census tracts in situations where no local participant existed or where state, local, or tribal governments' 'declined to participate. The primary purpose of census tracts is to provide a stable set of geographic units for the presentation of statistical data.\r\n\r\nCensus tracts generally have a population size between 1,200 and 8,000 people, with an optimum size of 4,000 people. A census tract usually covers a contiguous area; however, the spatial size of census tracts varies widely depending on the density of settlement. Census tract boundaries are delineated with the intention of being maintained over a long time so that statistical comparisons can be made from census to census. Census tracts occasionally are split due to population growth or merged as a result of substantial population decline.\r\n\r\nCensus tract boundaries generally follow visible and identifiable features. They may follow nonvisible legal boundaries, such as minor civil division (MCD) or incorporated place boundaries' diff --git a/server/lib/python/cartodb_services/cartodb_services/__init__.py b/server/lib/python/cartodb_services/cartodb_services/__init__.py index ef5caa4..68e03b5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/__init__.py @@ -34,4 +34,4 @@ def _reset(): plpy = None GD = None -from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder +from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py b/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py index ccb7d5f..f21ec9c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py +++ b/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py @@ -1,8 +1,8 @@ -from google import GoogleMapsBulkGeocoder -from here import HereMapsBulkGeocoder -from tomtom import TomTomBulkGeocoder -from mapbox import MapboxBulkGeocoder -from geocodio import GeocodioBulkGeocoder +from cartodb_services.google import GoogleMapsBulkGeocoder +from cartodb_services.here import HereMapsBulkGeocoder +from cartodb_services.tomtom import TomTomBulkGeocoder +from cartodb_services.mapbox import MapboxBulkGeocoder +from cartodb_services.geocodio import GeocodioBulkGeocoder BATCH_GEOCODER_CLASS_BY_PROVIDER = { 'google': GoogleMapsBulkGeocoder, diff --git a/server/lib/python/cartodb_services/cartodb_services/config/__init__.py b/server/lib/python/cartodb_services/cartodb_services/config/__init__.py index 6a98689..2a1c40f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/config/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/config/__init__.py @@ -1,3 +1,3 @@ -from service_configuration import ServiceConfiguration -from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter -from legacy_rate_limits import RateLimitsConfigLegacyBuilder +from .service_configuration import ServiceConfiguration +from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter +from .legacy_rate_limits import RateLimitsConfigLegacyBuilder diff --git a/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py index 11632f9..0a668a7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py +++ b/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py @@ -1,5 +1,5 @@ import json -from rate_limits import RateLimitsConfig +from cartodb_services.config.rate_limits import RateLimitsConfig class RateLimitsConfigLegacyBuilder(object): """ diff --git a/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py index 98f6a47..e09489a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py +++ b/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py @@ -1,6 +1,6 @@ import json -from service_configuration import ServiceConfiguration +from cartodb_services.config.service_configuration import ServiceConfiguration class RateLimitsConfig(object): """ diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index a2c250c..1e682a2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- -from tools import QuotaExceededException, Logger +from cartodb_services.tools import QuotaExceededException, Logger from collections import namedtuple import json @@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org except Exception as e: import sys logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname}) - metadata['processing_error'] = 'Error: {}'.format(e.message) + metadata['processing_error'] = 'Error: {}'.format(e) results.append([result[0], None, json.dumps(metadata)]) failed_count += 1 diff --git a/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py b/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py index 7b5bc90..05da24d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py @@ -1,2 +1,2 @@ -from geocoder import GeocodioGeocoder -from bulk_geocoder import GeocodioBulkGeocoder +from cartodb_services.geocodio.geocoder import GeocodioGeocoder +from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/google/__init__.py b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py index 7570e31..f249a66 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py @@ -1,2 +1,2 @@ -from geocoder import GoogleMapsGeocoder -from bulk_geocoder import GoogleMapsBulkGeocoder +from cartodb_services.google.geocoder import GoogleMapsGeocoder +from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index cde9f1c..107fb54 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -1,5 +1,5 @@ from multiprocessing import Pool -from exceptions import MalformedResult +from cartodb_services.google.exceptions import MalformedResult from cartodb_services import StreetPointBulkGeocoder from cartodb_services.geocoder import compose_address, geocoder_error_response from cartodb_services.google import GoogleMapsGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py b/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py index 9593f4f..6d936c8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py @@ -3,7 +3,7 @@ import googlemaps import base64 -from exceptions import InvalidGoogleCredentials +from cartodb_services.google.exceptions import InvalidGoogleCredentials class GoogleMapsClientFactory(): diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 30a62cf..0f96d5d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -1,9 +1,11 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- +try: + from urlparse import parse_qs +except: + from urllib.parse import parse_qs -from urlparse import parse_qs - -from exceptions import MalformedResult +from cartodb_services.google.exceptions import MalformedResult from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory diff --git a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py index 85cc596..a389e2a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py @@ -1,3 +1,3 @@ -from geocoder import HereMapsGeocoder -from bulk_geocoder import HereMapsBulkGeocoder -from routing import HereMapsRoutingIsoline +from cartodb_services.here.geocoder import HereMapsGeocoder +from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder +from cartodb_services.here.routing import HereMapsRoutingIsoline diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index fcc2e3e..ab36adf 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -1,7 +1,7 @@ import requests import json -from exceptions import WrongParams +from cartodb_services.here.exceptions import WrongParams from requests.adapters import HTTPAdapter from cartodb_services.metrics import Traceable diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py index a7afcc6..3d3416c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py @@ -1,4 +1,4 @@ -from routing import MapboxRouting, MapboxRoutingResponse -from geocoder import MapboxGeocoder -from bulk_geocoder import MapboxBulkGeocoder -from isolines import MapboxIsolines, MapboxIsochronesResponse +from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse +from cartodb_services.mapbox.geocoder import MapboxGeocoder +from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder +from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py index 831d4ad..c1d8cbc 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py @@ -1,5 +1,5 @@ -from routing import MapzenRouting, MapzenRoutingResponse -from isolines import MapzenIsolines -from geocoder import MapzenGeocoder -from matrix_client import MatrixClient -from isochrones import MapzenIsochrones +from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse +from cartodb_services.mapzen.isolines import MapzenIsolines +from cartodb_services.mapzen.geocoder import MapzenGeocoder +from cartodb_services.mapzen.matrix_client import MatrixClient +from cartodb_services.mapzen.isochrones import MapzenIsochrones diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index e6f51be..a6780f8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -135,7 +135,7 @@ class MapzenIsolines: # delete points that got None location_estimates_filtered = [] for i, c in enumerate(costs): - if c <> isorange: + if c != isorange: location_estimates_filtered.append(location_estimates[i]) return location_estimates_filtered diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index edca0ee..52cf7f3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,4 +1,4 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig -from quota import QuotaService -from user import UserMetricsService -from log import metrics, MetricsDataGatherer, Traceable +from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig +from cartodb_services.metrics.quota import QuotaService +from cartodb_services.metrics.user import UserMetricsService +from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index b762f3b..0c323f8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -45,7 +45,7 @@ class ServiceConfig(object): def _get_effective_monthly_quota(self, quota_key, default=0): quota_from_redis = self._redis_config.get(quota_key, None) - if quota_from_redis and quota_from_redis <> '': + if quota_from_redis and quota_from_redis != '': return int(quota_from_redis) else: return default diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 46b7fd2..bf66d18 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -6,7 +6,10 @@ import uuid import plpy from datetime import datetime from contextlib import contextmanager -from urlparse import urlparse +try: + from urlparse import urlparse +except: + from urllib.parse import urlparse @contextmanager diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 476318c..c532abb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -1,5 +1,5 @@ -from user import UserMetricsService -from log import MetricsDataGatherer +from cartodb_services.metrics.user import UserMetricsService +from cartodb_services.metrics.log import MetricsDataGatherer from datetime import date import re diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index 10dd21e..31e3db9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -1,5 +1,5 @@ from ..core.interfaces import ConfigBackendInterface -from null_config import NullConfigStorage +from cartodb_services.refactor.storage.null_config import NullConfigStorage class RedisConfigStorage(ConfigBackendInterface): diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py index a125af3..4350358 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py @@ -18,5 +18,5 @@ class RedisConnectionBuilder(): else: conn = StrictRedis(host=self._config.host, port=self._config.port, db=self._config.db, retry_on_timeout=True, - socket_timeout=self._config.timeout) + socket_timeout=self._config.timeout, decode_responses=True) return conn diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py index 21d5ced..34368d8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py @@ -1,4 +1,4 @@ -from geocoder import TomTomGeocoder -from bulk_geocoder import TomTomBulkGeocoder -from routing import TomTomRouting, TomTomRoutingResponse -from isolines import TomTomIsolines, TomTomIsochronesResponse +from cartodb_services.tomtom.geocoder import TomTomGeocoder +from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder +from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse +from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index 976469a..17e9e48 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1,9 +1,9 @@ -from redis_tools import RedisConnection, RedisDBConfig -from coordinates import Coordinate -from polyline import PolyLine -from log import Logger, LoggerConfig -from rate_limiter import RateLimiter -from service_manager import ServiceManager -from legacy_service_manager import LegacyServiceManager -from exceptions import QuotaExceededException, RateLimitExceeded -from country import country_to_iso3 +from .redis_tools import RedisConnection, RedisDBConfig +from .coordinates import Coordinate +from .polyline import PolyLine +from .log import Logger, LoggerConfig +from .rate_limiter import RateLimiter +from .service_manager import ServiceManager +from .legacy_service_manager import LegacyServiceManager +from .exceptions import QuotaExceededException, RateLimitExceeded +from .country import country_to_iso3 diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py index cd52f96..17bb1ca 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py @@ -1,7 +1,7 @@ import time import random from datetime import datetime -from exceptions import TimeoutException +from cartodb_services.tools.exceptions import TimeoutException import re DEFAULT_RETRY_TIMEOUT = 60 diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index e8b5800..48c8531 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -23,7 +23,7 @@ class RedisConnection: else: conn = StrictRedis(host=self._config.host, port=self._config.port, db=self._config.db, retry_on_timeout=True, - socket_timeout=self._config.timeout) + socket_timeout=self._config.timeout, decode_responses=True) return conn diff --git a/test/helpers/import_helper.py b/test/helpers/import_helper.py index 59f1b23..f15913f 100644 --- a/test/helpers/import_helper.py +++ b/test/helpers/import_helper.py @@ -20,7 +20,7 @@ class ImportHelper: response.raise_for_status() response_json = json.loads(response.text) if not response_json['success']: - print "Error importing the test dataset: {0}".format(response.text) + print("Error importing the test dataset: {0}".format(response.text)) sys.exit(1) while(True): table_name = ImportHelper.get_imported_table_name( @@ -42,8 +42,8 @@ class ImportHelper: schema, username, host, import_id, api_key) import_data_response = requests.get(import_url) if import_data_response.status_code != 200: - print "Error getting the table name from " \ - "the import data: {0}".format(import_data_response.text) + print("Error getting the table name from " \ + "the import data: {0}".format(import_data_response.text)) sys.exit(1) import_data_json = json.loads(import_data_response.text) @@ -57,5 +57,5 @@ class ImportHelper: ) response = requests.get(url) if response.status_code != 200: - print "Error cleaning the test dataset: {0}".format(response.text) + print("Error cleaning the test dataset: {0}".format(response.text)) sys.exit(1) diff --git a/test/helpers/integration_test_helper.py b/test/helpers/integration_test_helper.py index df050de..4d07382 100644 --- a/test/helpers/integration_test_helper.py +++ b/test/helpers/integration_test_helper.py @@ -10,7 +10,7 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) -def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005): +def assert_close_enough(xy_a, xy_b, rel_tol=0.001, abs_tol=0.0005): """ Asserts that the given points are "close enough", in a square. :param xy_a: Array of 2 elements, X and Y. @@ -23,7 +23,7 @@ def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005): assert_true(isclose(xy_a[i], xy_b[i], rel_tol, abs_tol), "Coord {} error: {} and {} are not closer than {}, {}".format( i, xy_a[i], xy_b[i], rel_tol, abs_tol - )) + )) class IntegrationTestHelper: @@ -45,11 +45,15 @@ class IntegrationTestHelper: } @classmethod - def execute_query_raw(cls, sql_api_url, query): + def execute_query_raw(cls, sql_api_url, query, method='GET'): requests.packages.urllib3.disable_warnings() - query_url = "{0}?q={1}".format(sql_api_url, query) - print "Executing query: {0}".format(query_url) - query_response = requests.get(query_url) + if method.upper() == 'GET': + query_url = "{0}?q={1}".format(sql_api_url, query) + query_response = requests.get(query_url) + print("Executing query GET: {0}".format(query_url)) + else: + query_response = requests.post(sql_api_url, data={"q": query}) + print("Executing query POST: {0}".format(sql_api_url)) if query_response.status_code != 200: raise Exception(json.loads(query_response.text)['error']) return json.loads(query_response.text) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index fd1b983..2350ff5 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -85,9 +85,9 @@ class TestStreetFunctionsSetUp(TestCase): TOMTOM_METADATAS = { 'Plaza España, Barcelona': - {'relevance': 0.85, 'precision': 'precise', 'match_types': ['street']}, + {'relevance': 0.72, 'precision': 'precise', 'match_types': ['street']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.45, 'precision': 'interpolated', 'match_types': ['street']} + {'relevance': 0.74, 'precision': 'precise', 'match_types': ['street']} } MAPBOX_METADATAS = { @@ -151,10 +151,9 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): query = "SELECT cdb_dataservices_client.cdb_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1".format(table) try: - IntegrationTestHelper.execute_query(self.sql_api_url, query) + self._run_authenticated(query)['rows'][0] except Exception as e: - assert_equal(e.message[0], - "permission denied for relation {}".format(table)) + assert_equal(e, "permission denied for relation {}".format(table)) def test_component_aggregation(self): query = "select st_x(the_geom), st_y(the_geom) from (" \ @@ -335,7 +334,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "{}" \ "]''::jsonb) as (cartodb_id integer, address text)', " \ "'address', null, null, null, {})".format(','.join(streets), batch_size) - response = self._run_authenticated(query) + response = self._run_authenticated(query, method='POST') assert_equal(n, len(response['rows'])) for row in response['rows']: assert_not_equal(row['st_x'], None) @@ -413,12 +412,14 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): for r, e in zip(response['rows'], expected): self.assert_metadata(r['metadata'], e) - def _run_authenticated(self, query): - authenticated_query = "{}&api_key={}".format(query, - self.env_variables[ - 'api_key']) - return IntegrationTestHelper.execute_query_raw(self.sql_api_url, - authenticated_query) + def _run_authenticated(self, query, method='GET'): + api_key = self.env_variables['api_key'] + url = self.sql_api_url + auth_query = "{}&api_key={}".format(query, api_key) + if method.upper() != 'GET': + auth_query = query + url = "{}?api_key={}".format(self.sql_api_url, api_key) + return IntegrationTestHelper.execute_query_raw(url, auth_query, method) @staticmethod def _x_y_by_cartodb_id(response): diff --git a/test/run_tests.py b/test/run_tests.py index 7308010..a505750 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -36,7 +36,7 @@ def main(): set_environment_variables(username, api_key, table_name, host, schema) execute_tests() except Exception as e: - print e.message + print(e) sys.exit(1) finally: clean_environment_variables() @@ -45,11 +45,11 @@ def main(): def usage(): - print """Usage: run_tests.py [options] username api_key + print("""Usage: run_tests.py [options] username api_key Options: -h: Show this help --host: take that host as base (by default is cartodb.com) - --schema: define the url schema [http/https] (by default https)""" + --schema: define the url schema [http/https] (by default https)""") def execute_tests(): @@ -59,7 +59,7 @@ def execute_tests(): stderr=subprocess.PIPE ) out, err = process.communicate() - print err + print(err.decode('utf-8')) regexp = re.compile(r'FAILED \(.*\)') if regexp.search(err) is not None: sys.exit(1)