cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'development' into tomtom_bulk_qps_retry

Browse Source
This commit is contained in:
Daniel García Aubert 2020-03-05 10:47:01 +01:00
commit 3560c07d7f
65 changed files with 328 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/main.yml vendored
View File

@ -32,6 +32,10 @@ jobs:
- name: Start docker-compose services
run: docker-compose -f docker-compose.yaml up -d
- name: Install required python3 libs
run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services"
if: env.PG_VERSION == 12
- name: Install required python libs
run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services"
@ -44,9 +48,9 @@ jobs:
timeout-minutes: 5
- name: Run server tests
run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck"
run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test_out/regression.diffs && false)"
timeout-minutes: 5
- name: Run client tests
run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck"
run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs && false)"
timeout-minutes: 5

34
client/Makefile
View File

@ -14,6 +14,14 @@ AWK = awk
PG_CONFIG = pg_config
PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
# PG12 compatibility
PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
PLPYTHONU := plpythonu
ifeq ($(PG_12_GE), true)
PLPYTHONU := plpython3u
endif
# OLD_VERSIONS = $(wildcard old_versions/*.sql)
# DATA = $(NEW_EXTENSION_ARTIFACT) \
# $(OLD_VERSIONS) \
@ -25,9 +33,9 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS)
SOURCES_DATA_DIR = sql/
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
TEST_DIR = test/
REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
TEST_DIR = test_out
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
# DATA is a special variable used by postgres build infrastructure
@ -54,7 +62,8 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES)
$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
rm -f $@
cat $(SOURCES_DATA_DIR)/*.sql >> $@
cat $(SOURCES_DATA_DIR)/*.sql | \
$(SED) -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
ifeq ($(PG_PARALLEL), 0)
# Remove PARALLEL in aggregates and functions
$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
@ -86,6 +95,7 @@ release: $(EXTENSION).control $(SOURCES_DATA)
devclean:
rm -f $(NEW_EXTENSION_ARTIFACT)
rm -f $(GENERATED_SQL_FILES)
rm -rf $(TEST_DIR)
# If needed remove PARALLEL tags from the release files
release_remove_parallel_deploy:
@ -97,9 +107,25 @@ ifeq ($(PG_PARALLEL), 0)
done
endif
# Replacing variables defined within test files
replace_variables:
mkdir -p $(TEST_DIR)
mkdir -p $(TEST_DIR)/expected
mkdir -p $(TEST_DIR)/sql
for f in $(REGRESS); do \
cat test/sql/$${f}.sql | \
sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
>> $(TEST_DIR)/sql/$${f}.sql; \
done
for f in $(REGRESS_EXPEC); do \
cat test/expected/$${f}.out | \
sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
>> $(TEST_DIR)/expected/$${f}.out; \
done
# Install the current release into the PostgreSQL extensions directory
deploy: release_remove_parallel_deploy
$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
install: deploy
install: replace_variables deploy

2
client/cdb_dataservices_client.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices client API extension'
default_version = '0.28.0'
default_version = '0.29.0'
requires = 'plproxy, cartodb'
superuser = true
schema = cdb_dataservices_client

13
client/sql/05_utils.sql
View File

@ -18,3 +18,16 @@ $func$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
$f$ LANGUAGE sql IMMUTABLE;
-- PG12_DEPRECATED
-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
CREATE TYPE cdb_dataservices_client.geomval AS (
geom geometry,
val double precision
);
END IF;
END$$;

4
client/sql/20_table_augmentation.sql
View File

@ -138,7 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeas
)
return True
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
username text,
@ -231,7 +231,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMea
fdw_server=plpy.quote_literal(server_name)))
return True
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable(
username text,

2
client/test/expected/00_installation_test.out
View File

@ -2,7 +2,7 @@
SET client_min_messages TO WARNING;
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION plpythonu;
CREATE EXTENSION @@plpythonu@@;
CREATE EXTENSION cartodb;
CREATE EXTENSION plproxy;
-- Install the extension

2
client/test/sql/00_installation_test.sql
View File

@ -2,7 +2,7 @@
SET client_min_messages TO WARNING;
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION plpythonu;
CREATE EXTENSION @@plpythonu@@;
CREATE EXTENSION cartodb;
CREATE EXTENSION plproxy;

60
server/extension/Makefile
View File

@ -1,11 +1,10 @@
# Makefile to generate the extension out of separate sql source files.
# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
EXTENSION = cdb_dataservices_server
EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
EXTVERSION = $(shell grep default_version $(EXTENSION).control.in | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
# The new version to be generated from templates
SED = sed
ERB = erb
REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
AWK = awk
@ -14,7 +13,8 @@ PG_CONFIG = pg_config
PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
TEST_DIR = test/
REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
TEST_DIR = test_out
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
# DATA is a special variable used by postgres build infrastructure
@ -27,15 +27,29 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS)
SOURCES_DATA_DIR = sql/
SOURCES_DATA = $(wildcard sql/*.sql)
SOURCES_DATA_DIR_OUT = sql_out/
# postgres build stuff
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
# PG12 compatibility
PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
PLPYTHONU := plpythonu
ifeq ($(PG_12_GE), true)
PLPYTHONU := plpython3u
endif
REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g'
ifneq ($(NEW_VERSION),)
REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g'
endif
$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
rm -f $@
cat $(SOURCES_DATA_DIR)/*.sql >> $@
cat $(SOURCES_DATA_DIR_OUT)/*.sql >> $@
ifeq ($(PG_PARALLEL), 0)
# Remove PARALLEL in aggregates and functions
$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
@ -44,16 +58,19 @@ ifeq ($(PG_PARALLEL), 0)
mv $(TMPFILE) $@
endif
$(EXTENSION).control: $(EXTENSION).control.in Makefile
$(SED) $(REPLACEMENTS) $< > $@
.PHONY: all
all: $(DATA)
all: replace_variables $(DATA)
.PHONY: release
release: $(EXTENSION).control $(SOURCES_DATA)
release: replace_variables $(EXTENSION).control $(SOURCES_DATA)
test -n "$(NEW_VERSION)" # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
git mv *.sql old_versions
$(SED) $(REPLACEMENTS) $(EXTENSION).control
$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
git add $(EXTENSION).control
cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
cat $(SOURCES_DATA_DIR_OUT)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
git add $(EXTENSION)--$(NEW_VERSION).sql
@ -66,6 +83,9 @@ release: $(EXTENSION).control $(SOURCES_DATA)
.PHONY: devclean
devclean:
rm -f $(NEW_EXTENSION_ARTIFACT)
rm -rf $(TEST_DIR)
rm -rf $(SOURCES_DATA_DIR_OUT)
rm -rf $(EXTENSION).control
# If needed remove PARALLEL tags from the release files
release_remove_parallel_deploy:
@ -82,6 +102,30 @@ deploy: release_remove_parallel_deploy
$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
# Replacing variables defined within test files
replace_variables:
# tests
mkdir -p $(TEST_DIR)
mkdir -p $(TEST_DIR)/expected
mkdir -p $(TEST_DIR)/sql
for f in $(REGRESS); do \
cat test/sql/$${f}.sql | \
sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
> $(TEST_DIR)/sql/$${f}.sql; \
done
for f in $(REGRESS_EXPEC); do \
cat test/expected/$${f}.out | \
sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
> $(TEST_DIR)/expected/$${f}.out; \
done
# data
mkdir -p $(SOURCES_DATA_DIR_OUT)
for f in $(notdir $(SOURCES_DATA)); do \
cat $(SOURCES_DATA_DIR)/$${f} | \
sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
> $(SOURCES_DATA_DIR_OUT)/$${f}; \
done
install: deploy
reinstall: install

4
server/extension/cdb_dataservices_server.control → server/extension/cdb_dataservices_server.control.in
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.37.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
default_version = '0.38.0'
requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

11
server/extension/sql/05_postgis3_adapter.sql Normal file
View File

@ -0,0 +1,11 @@
-- PG12_DEPRECATED
-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
CREATE TYPE cdb_dataservices_server.geomval AS (
geom geometry,
val double precision
);
END IF;
END$$;

6
server/extension/sql/100_routing_helper.sql
View File

@ -69,7 +69,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
raise Exception('Error trying to calculate Mapbox routing')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
username TEXT,
@ -137,7 +137,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
raise Exception('Error trying to calculate TomTom routing')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
username TEXT,
@ -198,4 +198,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
raise Exception('Error trying to calculate mapzen routing')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

4
server/extension/sql/105_route_between_points.sql
View File

@ -37,7 +37,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
@ -76,4 +76,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

2
server/extension/sql/10_redis_helper.sql
View File

@ -15,4 +15,4 @@ RETURNS boolean AS $$
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

30
server/extension/sql/110_data_observatory_augmentation.sql
View File

@ -14,7 +14,7 @@ RETURNS text AS $$
user_obs_config = GD["user_obs_config_{0}".format(username)]
return user_obs_config.connection_str
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
username TEXT,
@ -68,7 +68,7 @@ RETURNS json AS $$
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
username TEXT,
@ -125,7 +125,7 @@ RETURNS SETOF JSON AS $$
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
username TEXT,
@ -177,7 +177,7 @@ RETURNS json AS $$
raise Exception('Error trying to obs_get_segment_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
username TEXT,
@ -232,7 +232,7 @@ RETURNS SETOF JSON AS $$
raise Exception('Error trying to OBS_GetSegmentSnapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
username TEXT,
@ -289,7 +289,7 @@ RETURNS NUMERIC AS $$
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
username TEXT,
@ -344,7 +344,7 @@ RETURNS TEXT AS $$
raise Exception('Error trying to OBS_GetCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
username TEXT,
@ -401,7 +401,7 @@ RETURNS NUMERIC AS $$
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
username TEXT,
@ -456,7 +456,7 @@ RETURNS TEXT AS $$
raise Exception('Error trying to OBS_GetUSCensusCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
username TEXT,
@ -511,7 +511,7 @@ RETURNS NUMERIC AS $$
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
username TEXT,
@ -566,7 +566,7 @@ RETURNS NUMERIC AS $$
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
username TEXT,
@ -627,7 +627,7 @@ RETURNS TABLE (
raise Exception('Error trying to OBS_GetData')
finally:
quota_service.increment_total_service_use(len(geomvals))
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
username TEXT,
@ -688,7 +688,7 @@ RETURNS TABLE (
raise Exception('Error trying to OBS_GetData')
finally:
quota_service.increment_total_service_use(len(geomrefs))
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta(
username TEXT,
@ -736,7 +736,7 @@ RETURNS JSON AS $$
import sys
logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeta')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation(
username TEXT,
@ -782,4 +782,4 @@ RETURNS TABLE(valid boolean, errors text[]) AS $$
import sys
logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_MetadataValidation')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

4
server/extension/sql/115_data_observatory_exploration.sql
View File

@ -55,7 +55,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
raise Exception('Error trying to OBS_Search')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
username TEXT,
@ -113,4 +113,4 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

14
server/extension/sql/120_data_observatory_geometries.sql
View File

@ -49,7 +49,7 @@ RETURNS geometry(Geometry, 4326) AS $$
raise Exception('Error trying to OBS_GetBoundary')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
username TEXT,
@ -102,7 +102,7 @@ RETURNS TEXT AS $$
raise Exception('Error trying to OBS_GetBoundaryId')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
username TEXT,
@ -155,7 +155,7 @@ RETURNS geometry(Geometry, 4326) AS $$
raise Exception('Error trying to OBS_GetBoundaryById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
username TEXT,
@ -215,7 +215,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
username TEXT,
@ -277,7 +277,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
username TEXT,
@ -337,7 +337,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
username TEXT,
@ -399,4 +399,4 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

2
server/extension/sql/125_data_observatory_table_augment.sql
View File

@ -8,7 +8,7 @@ RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$

12
server/extension/sql/15_config_helper.sql
View File

@ -8,7 +8,7 @@ RETURNS boolean AS $$
logger_config = LoggerConfig(plpy)
GD[cache_key] = logger_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
-- This is done in order to avoid an undesired depedency on cartodb extension
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text)
@ -46,7 +46,7 @@ RETURNS boolean AS $$
geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider)
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
@ -60,7 +60,7 @@ RETURNS boolean AS $$
geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
RETURNS boolean AS $$
@ -74,7 +74,7 @@ RETURNS boolean AS $$
isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = isolines_routing_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
RETURNS boolean AS $$
@ -88,7 +88,7 @@ RETURNS boolean AS $$
routing_config = RoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = routing_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text)
RETURNS boolean AS $$
@ -102,4 +102,4 @@ RETURNS boolean AS $$
obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = obs_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

4
server/extension/sql/200_quotas.sql
View File

@ -106,7 +106,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
@ -136,7 +136,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota(
username TEXT,

26
server/extension/sql/20_geocode_street.sql
View File

@ -36,7 +36,7 @@ RETURNS Geometry AS $$
else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
@ -52,7 +52,7 @@ RETURNS Geometry AS $$
else:
raise Exception('Here geocoder is not available for your account.')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -67,7 +67,7 @@ RETURNS Geometry AS $$
else:
raise Exception('Google geocoder is not available for your account.')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -80,7 +80,7 @@ RETURNS Geometry AS $$
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -93,7 +93,7 @@ RETURNS Geometry AS $$
mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -106,7 +106,7 @@ RETURNS Geometry AS $$
tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -119,7 +119,7 @@ RETURNS Geometry AS $$
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -152,7 +152,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode street point using here maps')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -184,7 +184,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode street point using google maps')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -225,7 +225,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode street point using mapzen')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -271,7 +271,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode street point using Mapbox')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -317,7 +317,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode street point using TomTom')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
@ -363,4 +363,4 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode street point using Geocodio')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

8
server/extension/sql/210_rates.sql
View File

@ -16,7 +16,7 @@ RETURNS JSON AS $$
return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period})
else:
return None
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit(
username TEXT,
@ -40,7 +40,7 @@ RETURNS VOID AS $$
period = None
config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
config_setter.set_user_rate_limits(config)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit(
username TEXT,
@ -64,7 +64,7 @@ RETURNS VOID AS $$
period = None
config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
config_setter.set_org_rate_limits(config)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit(
username TEXT,
@ -88,4 +88,4 @@ RETURNS VOID AS $$
period = None
config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
config_setter.set_server_rate_limits(config)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;

12
server/extension/sql/21_bulk_geocode_street.sql
View File

@ -40,7 +40,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@ -51,7 +51,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@ -62,7 +62,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@ -79,7 +79,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@ -96,7 +96,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD)
geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@ -113,4 +113,4 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

2
server/extension/sql/30_admin0.sql
View File

@ -35,7 +35,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode admin0 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
--------------------------------------------------------------------------------

4
server/extension/sql/40_admin1.sql
View File

@ -36,7 +36,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
@ -74,7 +74,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
--------------------------------------------------------------------------------

14
server/extension/sql/50_namedplaces.sql
View File

@ -1,7 +1,7 @@
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
import spiexceptions
from plpy import spiexceptions
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@ -16,12 +16,12 @@ RETURNS Geometry AS $$
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
import spiexceptions
from plpy import spiexceptions
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@ -36,12 +36,12 @@ RETURNS Geometry AS $$
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
import spiexceptions
from plpy import spiexceptions
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@ -56,7 +56,7 @@ RETURNS Geometry AS $$
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
RETURNS Geometry AS $$
@ -101,7 +101,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
--------------------------------------------------------------------------------

8
server/extension/sql/60_postalcodes.sql
View File

@ -35,7 +35,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision)
RETURNS Geometry AS $$
@ -77,7 +77,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text)
RETURNS Geometry AS $$
@ -119,7 +119,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision)
RETURNS Geometry AS $$
@ -162,7 +162,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text)
RETURNS Geometry AS $$

2
server/extension/sql/70_ips.sql
View File

@ -35,7 +35,7 @@ RETURNS Geometry AS $$
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
--------------------------------------------------------------------------------

14
server/extension/sql/80_isolines_helper.sql
View File

@ -55,7 +55,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance(
username TEXT,
@ -121,7 +121,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
username TEXT,
@ -185,7 +185,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('Error trying to get Mapbox isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance(
username TEXT,
@ -249,7 +249,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('Error trying to get TomTom isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
username TEXT,
@ -310,7 +310,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('Error trying to get mapzen isochrones')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
username TEXT,
@ -368,7 +368,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('Error trying to get Mapbox isochrones')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
username TEXT,
@ -426,4 +426,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('Error trying to get TomTom isochrones')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

10
server/extension/sql/85_isodistance.sql
View File

@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
else:
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- heremaps isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- mapzen isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -60,7 +60,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- mapbox isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -74,7 +74,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- tomtom isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -88,4 +88,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

10
server/extension/sql/90_isochrone.sql
View File

@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
else:
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- heremaps isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- mapzen isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -59,7 +59,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- mapbox isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -72,7 +72,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
-- tomtom isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@ -85,4 +85,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

2
server/extension/test/expected/00_install_test.out
View File

@ -2,7 +2,7 @@
SET client_min_messages TO WARNING;
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION plpythonu;
CREATE EXTENSION @@plpythonu@@;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;

1
server/extension/test/expected/100_data_observatory_test.out
View File

@ -1,3 +1,4 @@
\set ECHO none
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

2
server/extension/test/sql/00_install_test.sql
View File

@ -2,7 +2,7 @@
SET client_min_messages TO WARNING;
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION plpythonu;
CREATE EXTENSION @@plpythonu@@;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;

7
server/extension/test/sql/100_data_observatory_test.sql
View File

@ -1,3 +1,10 @@
\set ECHO none
-- add the schema cdb_dataservices_server to the SEARCH_PATH
DO $$ BEGIN
PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false);
END $$;
\set ECHO all
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

9
server/extension/test/sql/366_empty_table_test.sql
View File

@ -2,6 +2,11 @@
\set VERBOSITY verbose
SET client_min_messages TO error;
-- add the schema cdb_dataservices_server to the SEARCH_PATH
DO $$ BEGIN
PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false);
END $$;
-- Set configuration for a user 'foo'
DO $$
import json
@ -14,7 +19,7 @@ DO $$
service_config.user.set('soft_obs_general_limit', True)
service_config.user.set('period_end_date', '20170516')
$$ LANGUAGE plpythonu;
$$ LANGUAGE @@plpythonu@@;
-- Mock Observatory backend function
@ -37,7 +42,7 @@ GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api;
SELECT * FROM cdb_dataservices_server.OBS_GetData(
'foo',
NULL,
'{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::public._geomval,
'{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::_geomval,
'[{"id": 1, "score": 52.7515548093083898758340051256007949661290516400338, "geom_id": "us.census.tiger.census_tract", "denom_id": "us.census.acs.B01003001", "numer_id": "us.census.acs.B03002003", "geom_name": "US Census Tracts", "geom_type": "Geometry", "num_geoms": 2.86483076549783307739486952736, "denom_name": "Total Population", "denom_type": "Numeric", "numer_name": "White Population", "numer_type": "Numeric", "score_rank": 1, "target_area": 0.000307374806576033, "geom_colname": "the_geom", "score_rownum": 1, "target_geoms": null, "denom_colname": "total_pop", "denom_reltype": '
'"denominator", "geom_timespan": "2015", "normalization": "prenormalized", "numer_colname": "white_pop", "timespan_rank": 1, "geom_tablename": "obs_87a814e485deabe3b12545a537f693d16ca702c2", "max_score_rank": null, "numer_timespan": "2010 - 2014", "suggested_name": "white_pop_2010_2014", "denom_aggregate": "sum", "denom_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "numer_aggregate": "sum", "numer_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "timespan_rownum": 1, "geom_description": "Census tracts are small, relatively permanent statistical subdivisions of a county or equivalent entity that are updated by local participants prior to each decennial census as part of the Census Bureau’s Participant Statistical Areas Program. The Census Bureau delineates census tracts in situations where no local participant existed or where state, local, or tribal governments'
'declined to participate. The primary purpose of census tracts is to provide a stable set of geographic units for the presentation of statistical data.\r\n\r\nCensus tracts generally have a population size between 1,200 and 8,000 people, with an optimum size of 4,000 people. A census tract usually covers a contiguous area; however, the spatial size of census tracts varies widely depending on the density of settlement. Census tract boundaries are delineated with the intention of being maintained over a long time so that statistical comparisons can be made from census to census. Census tracts occasionally are split due to population growth or merged as a result of substantial population decline.\r\n\r\nCensus tract boundaries generally follow visible and identifiable features. They may follow nonvisible legal boundaries, such as minor civil division (MCD) or incorporated place boundaries'

2
server/lib/python/cartodb_services/cartodb_services/__init__.py
View File

@ -34,4 +34,4 @@ def _reset():
plpy = None
GD = None
from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder
from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder

10
server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py
View File

@ -1,8 +1,8 @@
from google import GoogleMapsBulkGeocoder
from here import HereMapsBulkGeocoder
from tomtom import TomTomBulkGeocoder
from mapbox import MapboxBulkGeocoder
from geocodio import GeocodioBulkGeocoder
from cartodb_services.google import GoogleMapsBulkGeocoder
from cartodb_services.here import HereMapsBulkGeocoder
from cartodb_services.tomtom import TomTomBulkGeocoder
from cartodb_services.mapbox import MapboxBulkGeocoder
from cartodb_services.geocodio import GeocodioBulkGeocoder
BATCH_GEOCODER_CLASS_BY_PROVIDER = {
'google': GoogleMapsBulkGeocoder,

6
server/lib/python/cartodb_services/cartodb_services/config/__init__.py
View File

@ -1,3 +1,3 @@
from service_configuration import ServiceConfiguration
from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
from legacy_rate_limits import RateLimitsConfigLegacyBuilder
from .service_configuration import ServiceConfiguration
from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
from .legacy_rate_limits import RateLimitsConfigLegacyBuilder

2
server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py
View File

@ -1,5 +1,5 @@
import json
from rate_limits import RateLimitsConfig
from cartodb_services.config.rate_limits import RateLimitsConfig
class RateLimitsConfigLegacyBuilder(object):
"""

2
server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py
View File

@ -1,6 +1,6 @@
import json
from service_configuration import ServiceConfiguration
from cartodb_services.config.service_configuration import ServiceConfiguration
class RateLimitsConfig(object):
"""

4
server/lib/python/cartodb_services/cartodb_services/geocoder.py
View File

@ -1,7 +1,7 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
from tools import QuotaExceededException, Logger
from cartodb_services.tools import QuotaExceededException, Logger
from collections import namedtuple
import json
@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
except Exception as e:
import sys
logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname})
metadata['processing_error'] = 'Error: {}'.format(e.message)
metadata['processing_error'] = 'Error: {}'.format(e)
results.append([result[0], None, json.dumps(metadata)])
failed_count += 1

4
server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py
View File

@ -1,2 +1,2 @@
from geocoder import GeocodioGeocoder
from bulk_geocoder import GeocodioBulkGeocoder
from cartodb_services.geocodio.geocoder import GeocodioGeocoder
from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder

4
server/lib/python/cartodb_services/cartodb_services/google/__init__.py
View File

@ -1,2 +1,2 @@
from geocoder import GoogleMapsGeocoder
from bulk_geocoder import GoogleMapsBulkGeocoder
from cartodb_services.google.geocoder import GoogleMapsGeocoder
from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder

2
server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py
View File

@ -1,5 +1,5 @@
from multiprocessing import Pool
from exceptions import MalformedResult
from cartodb_services.google.exceptions import MalformedResult
from cartodb_services import StreetPointBulkGeocoder
from cartodb_services.geocoder import compose_address, geocoder_error_response
from cartodb_services.google import GoogleMapsGeocoder

2
server/lib/python/cartodb_services/cartodb_services/google/client_factory.py
View File

@ -3,7 +3,7 @@
import googlemaps
import base64
from exceptions import InvalidGoogleCredentials
from cartodb_services.google.exceptions import InvalidGoogleCredentials
class GoogleMapsClientFactory():

8
server/lib/python/cartodb_services/cartodb_services/google/geocoder.py
View File

@ -1,9 +1,11 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
try:
from urlparse import parse_qs
except:
from urllib.parse import parse_qs
from urlparse import parse_qs
from exceptions import MalformedResult
from cartodb_services.google.exceptions import MalformedResult
from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE
from cartodb_services.google.exceptions import InvalidGoogleCredentials
from client_factory import GoogleMapsClientFactory

6
server/lib/python/cartodb_services/cartodb_services/here/__init__.py
View File

@ -1,3 +1,3 @@
from geocoder import HereMapsGeocoder
from bulk_geocoder import HereMapsBulkGeocoder
from routing import HereMapsRoutingIsoline
from cartodb_services.here.geocoder import HereMapsGeocoder
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
from cartodb_services.here.routing import HereMapsRoutingIsoline

2
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@ -1,7 +1,7 @@
import requests
import json
from exceptions import WrongParams
from cartodb_services.here.exceptions import WrongParams
from requests.adapters import HTTPAdapter
from cartodb_services.metrics import Traceable

8
server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py
View File

@ -1,4 +1,4 @@
from routing import MapboxRouting, MapboxRoutingResponse
from geocoder import MapboxGeocoder
from bulk_geocoder import MapboxBulkGeocoder
from isolines import MapboxIsolines, MapboxIsochronesResponse
from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse
from cartodb_services.mapbox.geocoder import MapboxGeocoder
from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder
from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse

10
server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py
View File

@ -1,5 +1,5 @@
from routing import MapzenRouting, MapzenRoutingResponse
from isolines import MapzenIsolines
from geocoder import MapzenGeocoder
from matrix_client import MatrixClient
from isochrones import MapzenIsochrones
from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse
from cartodb_services.mapzen.isolines import MapzenIsolines
from cartodb_services.mapzen.geocoder import MapzenGeocoder
from cartodb_services.mapzen.matrix_client import MatrixClient
from cartodb_services.mapzen.isochrones import MapzenIsochrones

2
server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py
View File

@ -135,7 +135,7 @@ class MapzenIsolines:
# delete points that got None
location_estimates_filtered = []
for i, c in enumerate(costs):
if c <> isorange:
if c != isorange:
location_estimates_filtered.append(location_estimates[i])
return location_estimates_filtered

8
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
View File

@ -1,4 +1,4 @@
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
from quota import QuotaService
from user import UserMetricsService
from log import metrics, MetricsDataGatherer, Traceable
from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
from cartodb_services.metrics.quota import QuotaService
from cartodb_services.metrics.user import UserMetricsService
from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable

2
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -45,7 +45,7 @@ class ServiceConfig(object):
def _get_effective_monthly_quota(self, quota_key, default=0):
quota_from_redis = self._redis_config.get(quota_key, None)
if quota_from_redis and quota_from_redis <> '':
if quota_from_redis and quota_from_redis != '':
return int(quota_from_redis)
else:
return default

5
server/lib/python/cartodb_services/cartodb_services/metrics/log.py
View File

@ -6,7 +6,10 @@ import uuid
import plpy
from datetime import datetime
from contextlib import contextmanager
from urlparse import urlparse
try:
from urlparse import urlparse
except:
from urllib.parse import urlparse
@contextmanager

4
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@ -1,5 +1,5 @@
from user import UserMetricsService
from log import MetricsDataGatherer
from cartodb_services.metrics.user import UserMetricsService
from cartodb_services.metrics.log import MetricsDataGatherer
from datetime import date
import re

2
server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py
View File

@ -1,5 +1,5 @@
from ..core.interfaces import ConfigBackendInterface
from null_config import NullConfigStorage
from cartodb_services.refactor.storage.null_config import NullConfigStorage
class RedisConfigStorage(ConfigBackendInterface):

2
server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py
View File

@ -18,5 +18,5 @@ class RedisConnectionBuilder():
else:
conn = StrictRedis(host=self._config.host, port=self._config.port,
db=self._config.db, retry_on_timeout=True,
socket_timeout=self._config.timeout)
socket_timeout=self._config.timeout, decode_responses=True)
return conn

8
server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py
View File

@ -1,4 +1,4 @@
from geocoder import TomTomGeocoder
from bulk_geocoder import TomTomBulkGeocoder
from routing import TomTomRouting, TomTomRoutingResponse
from isolines import TomTomIsolines, TomTomIsochronesResponse
from cartodb_services.tomtom.geocoder import TomTomGeocoder
from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder
from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse
from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse

18
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py
View File

@ -1,9 +1,9 @@
from redis_tools import RedisConnection, RedisDBConfig
from coordinates import Coordinate
from polyline import PolyLine
from log import Logger, LoggerConfig
from rate_limiter import RateLimiter
from service_manager import ServiceManager
from legacy_service_manager import LegacyServiceManager
from exceptions import QuotaExceededException, RateLimitExceeded
from country import country_to_iso3
from .redis_tools import RedisConnection, RedisDBConfig
from .coordinates import Coordinate
from .polyline import PolyLine
from .log import Logger, LoggerConfig
from .rate_limiter import RateLimiter
from .service_manager import ServiceManager
from .legacy_service_manager import LegacyServiceManager
from .exceptions import QuotaExceededException, RateLimitExceeded
from .country import country_to_iso3

2
server/lib/python/cartodb_services/cartodb_services/tools/qps.py
View File

@ -1,7 +1,7 @@
import time
import random
from datetime import datetime
from exceptions import TimeoutException
from cartodb_services.tools.exceptions import TimeoutException
import re
DEFAULT_RETRY_TIMEOUT = 60

2
server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py
View File

@ -23,7 +23,7 @@ class RedisConnection:
else:
conn = StrictRedis(host=self._config.host, port=self._config.port,
db=self._config.db, retry_on_timeout=True,
socket_timeout=self._config.timeout)
socket_timeout=self._config.timeout, decode_responses=True)
return conn

8
test/helpers/import_helper.py
View File

@ -20,7 +20,7 @@ class ImportHelper:
response.raise_for_status()
response_json = json.loads(response.text)
if not response_json['success']:
print "Error importing the test dataset: {0}".format(response.text)
print("Error importing the test dataset: {0}".format(response.text))
sys.exit(1)
while(True):
table_name = ImportHelper.get_imported_table_name(
@ -42,8 +42,8 @@ class ImportHelper:
schema, username, host, import_id, api_key)
import_data_response = requests.get(import_url)
if import_data_response.status_code != 200:
print "Error getting the table name from " \
"the import data: {0}".format(import_data_response.text)
print("Error getting the table name from " \
"the import data: {0}".format(import_data_response.text))
sys.exit(1)
import_data_json = json.loads(import_data_response.text)
@ -57,5 +57,5 @@ class ImportHelper:
)
response = requests.get(url)
if response.status_code != 200:
print "Error cleaning the test dataset: {0}".format(response.text)
print("Error cleaning the test dataset: {0}".format(response.text))
sys.exit(1)

16
test/helpers/integration_test_helper.py
View File

@ -10,7 +10,7 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005):
def assert_close_enough(xy_a, xy_b, rel_tol=0.001, abs_tol=0.0005):
"""
Asserts that the given points are "close enough", in a square.
:param xy_a: Array of 2 elements, X and Y.
@ -23,7 +23,7 @@ def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005):
assert_true(isclose(xy_a[i], xy_b[i], rel_tol, abs_tol),
"Coord {} error: {} and {} are not closer than {}, {}".format(
i, xy_a[i], xy_b[i], rel_tol, abs_tol
))
))
class IntegrationTestHelper:
@ -45,11 +45,15 @@ class IntegrationTestHelper:
}
@classmethod
def execute_query_raw(cls, sql_api_url, query):
def execute_query_raw(cls, sql_api_url, query, method='GET'):
requests.packages.urllib3.disable_warnings()
query_url = "{0}?q={1}".format(sql_api_url, query)
print "Executing query: {0}".format(query_url)
query_response = requests.get(query_url)
if method.upper() == 'GET':
query_url = "{0}?q={1}".format(sql_api_url, query)
query_response = requests.get(query_url)
print("Executing query GET: {0}".format(query_url))
else:
query_response = requests.post(sql_api_url, data={"q": query})
print("Executing query POST: {0}".format(sql_api_url))
if query_response.status_code != 200:
raise Exception(json.loads(query_response.text)['error'])
return json.loads(query_response.text)

25
test/integration/test_street_functions.py
View File

@ -85,9 +85,9 @@ class TestStreetFunctionsSetUp(TestCase):
TOMTOM_METADATAS = {
'Plaza España, Barcelona':
{'relevance': 0.85, 'precision': 'precise', 'match_types': ['street']},
{'relevance': 0.72, 'precision': 'precise', 'match_types': ['street']},
'Santiago Rusiñol 123, Valladolid':
{'relevance': 0.45, 'precision': 'interpolated', 'match_types': ['street']}
{'relevance': 0.74, 'precision': 'precise', 'match_types': ['street']}
}
MAPBOX_METADATAS = {
@ -151,10 +151,9 @@ class TestStreetFunctions(TestStreetFunctionsSetUp):
query = "SELECT cdb_dataservices_client.cdb_geocode_street_point(street) " \
"as geometry FROM {0} LIMIT 1".format(table)
try:
IntegrationTestHelper.execute_query(self.sql_api_url, query)
self._run_authenticated(query)['rows'][0]
except Exception as e:
assert_equal(e.message[0],
"permission denied for relation {}".format(table))
assert_equal(e, "permission denied for relation {}".format(table))
def test_component_aggregation(self):
query = "select st_x(the_geom), st_y(the_geom) from (" \
@ -335,7 +334,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
"{}" \
"]''::jsonb) as (cartodb_id integer, address text)', " \
"'address', null, null, null, {})".format(','.join(streets), batch_size)
response = self._run_authenticated(query)
response = self._run_authenticated(query, method='POST')
assert_equal(n, len(response['rows']))
for row in response['rows']:
assert_not_equal(row['st_x'], None)
@ -413,12 +412,14 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
for r, e in zip(response['rows'], expected):
self.assert_metadata(r['metadata'], e)
def _run_authenticated(self, query):
authenticated_query = "{}&api_key={}".format(query,
self.env_variables[
'api_key'])
return IntegrationTestHelper.execute_query_raw(self.sql_api_url,
authenticated_query)
def _run_authenticated(self, query, method='GET'):
api_key = self.env_variables['api_key']
url = self.sql_api_url
auth_query = "{}&api_key={}".format(query, api_key)
if method.upper() != 'GET':
auth_query = query
url = "{}?api_key={}".format(self.sql_api_url, api_key)
return IntegrationTestHelper.execute_query_raw(url, auth_query, method)
@staticmethod
def _x_y_by_cartodb_id(response):

8
test/run_tests.py
View File

@ -36,7 +36,7 @@ def main():
set_environment_variables(username, api_key, table_name, host, schema)
execute_tests()
except Exception as e:
print e.message
print(e)
sys.exit(1)
finally:
clean_environment_variables()
@ -45,11 +45,11 @@ def main():
def usage():
print """Usage: run_tests.py [options] username api_key
print("""Usage: run_tests.py [options] username api_key
Options:
-h: Show this help
--host: take that host as base (by default is cartodb.com)
--schema: define the url schema [http/https] (by default https)"""
--schema: define the url schema [http/https] (by default https)""")
def execute_tests():
@ -59,7 +59,7 @@ def execute_tests():
stderr=subprocess.PIPE
)
out, err = process.communicate()
print err
print(err.decode('utf-8'))
regexp = re.compile(r'FAILED \(.*\)')
if regexp.search(err) is not None:
sys.exit(1)