cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Data Observatory functions

Browse Source
This commit is contained in:
Jesús Arroyo Torrens 2021-03-03 16:22:38 +01:00
parent 2b98908dbc
commit f4d8ebab44
15 changed files with 6 additions and 2092 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
NEWS.md
View File

@ -1,3 +1,8 @@
Mar 4th, 2021
=============
* Version `0.39.4` of the server extension
* Remove Data Observatory v1
Jan 26th, 2021 Jan 26th, 2021
============== ==============
* Version `0.39.3` of the server extension * Version `0.39.3` of the server extension

31
README.md
View File

@ -80,28 +80,6 @@ Steps to deploy a new Data Services API version :
psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS cdb_geocoder; COMMIT" -e psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS cdb_geocoder; COMMIT" -e
``` ```
- [optional] install data observatory extension
- Make the extension available in postgresql to be installed
```
git clone https://github.com/CartoDB/observatory-extension.git
cd observatory
sudo make install
```
- This extension needs data, dumps are not available so we're going to use the test fixtures to make it work. Execute:
```
psql -U postgres -d dataservices_db -f src/pg/test/fixtures/load_fixtures.sql
```
- Give permission to execute and select to the `geocoder_api` user:
```
psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory VERSION 'dev'; COMMIT" -e
psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO geocoder_api; COMMIT" -e
psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api; COMMIT" -e
psql -U postgres -d dataservices_db -c "BEGIN;GRANT USAGE ON SCHEMA cdb_observatory TO geocoder_api; COMMIT" -e
psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO geocoder_api; COMMIT" -e
psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA observatory TO geocoder_api; COMMIT" -e
psql -U postgres -d dataservices_db -c "BEGIN;GRANT USAGE ON SCHEMA observatory TO geocoder_api; COMMIT" -e
```
### Server configuration ### Server configuration
Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`. Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`.
@ -190,15 +168,6 @@ SELECT CDB_Conf_SetConf(
); );
``` ```
#### Data Observatory
```sql
SELECT CDB_Conf_SetConf(
'data_observatory_conf',
'{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'
);
```
#### Logger #### Logger
```sql ```sql

2
server/extension/Makefile
View File

@ -1,7 +1,7 @@
# Makefile to generate the extension out of separate sql source files. # Makefile to generate the extension out of separate sql source files.
# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
EXTENSION = cdb_dataservices_server EXTENSION = cdb_dataservices_server
EXTVERSION = 0.39.3 EXTVERSION = 0.39.4
# The new version to be generated from templates # The new version to be generated from templates
SED ?= sed SED ?= sed

785
server/extension/sql/110_data_observatory_augmentation.sql
View File

@ -1,785 +0,0 @@
--
-- Observatory connection config
--
-- The purpose of this function is provide to the PL/Proxy functions
-- the connection string needed to connect with the current production database
CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
username TEXT,
orgname TEXT)
RETURNS text AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
return user_obs_config.connection_str
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_segment_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetSegmentSnapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
category_id TEXT,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
category_id TEXT,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getcategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getuscensusmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getuscensuscategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getpopulation', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['population']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
username TEXT,
orgname TEXT,
geom_ref TEXT,
measure_id TEXT,
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
username TEXT,
orgname TEXT,
geom_ref TEXT,
measure_id TEXT,
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getmeasurebyid', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
username TEXT,
orgname TEXT,
geomvals geomval[],
params JSON,
merge BOOLEAN DEFAULT True)
RETURNS TABLE (
id INT,
data JSON
) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
username TEXT,
orgname TEXT,
geomvals geomval[],
params JSON,
merge BOOLEAN DEFAULT True)
RETURNS TABLE (
id INT,
data JSON
) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdata', user_obs_config, logger, params):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"])
result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge])
empty_results = len(geomvals) - len(result)
if empty_results > 0:
quota_service.increment_empty_service_use(empty_results)
if result:
quota_service.increment_success_service_use(len(result))
return result
else:
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use(len(geomvals))
logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetData')
finally:
quota_service.increment_total_service_use(len(geomvals))
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
username TEXT,
orgname TEXT,
geomrefs TEXT[],
params JSON)
RETURNS TABLE (
id TEXT,
data JSON
) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
username TEXT,
orgname TEXT,
geomrefs TEXT[],
params JSON)
RETURNS TABLE (
id TEXT,
data JSON
) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdata', user_obs_config, logger, params):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"])
result = plpy.execute(obs_plan, [username, orgname, geomrefs, params])
empty_results = len(geomrefs) - len(result)
if empty_results > 0:
quota_service.increment_empty_service_use(empty_results)
if result:
quota_service.increment_success_service_use(len(result))
return result
else:
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use(len(geomrefs))
exc_info = sys.exc_info()
logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2]))
logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetData')
finally:
quota_service.increment_total_service_use(len(geomrefs))
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta(
username TEXT,
orgname TEXT,
geom Geometry(Geometry, 4326),
params JSON,
max_timespan_rank INTEGER DEFAULT NULL,
max_score_rank INTEGER DEFAULT NULL,
target_geoms INTEGER DEFAULT NULL)
RETURNS JSON AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta(
username TEXT,
orgname TEXT,
geom Geometry(Geometry, 4326),
params JSON,
max_timespan_rank INTEGER DEFAULT NULL,
max_score_rank INTEGER DEFAULT NULL,
target_geoms INTEGER DEFAULT NULL)
RETURNS JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
with metrics('obs_getmeta', user_obs_config, logger, params):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"])
result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms])
if result:
return result[0]['meta']
else:
return None
except BaseException as e:
import sys
logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeta')
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation(
username TEXT,
orgname TEXT,
geometry_extent Geometry(Geometry, 4326),
geometry_type text,
params JSON,
target_geoms INTEGER DEFAULT NULL)
RETURNS TABLE(valid boolean, errors text[]) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation(
username TEXT,
orgname TEXT,
geometry_extent Geometry(Geometry, 4326),
geometry_type text,
params JSON,
target_geoms INTEGER DEFAULT NULL)
RETURNS TABLE(valid boolean, errors text[]) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
with metrics('obs_metadatavalidation', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"])
result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms])
if result:
return result
else:
return []
except BaseException as e:
import sys
logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_MetadataValidation')
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

116
server/extension/sql/115_data_observatory_exploration.sql
View File

@ -1,116 +0,0 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search(
username TEXT,
orgname TEXT,
search_term TEXT,
relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
username TEXT,
orgname TEXT,
search_term TEXT,
relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_search', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
if result:
resp = []
for element in result:
id = element['id']
description = element['description']
name = element['name']
aggregate = element['aggregate']
source = element['source']
resp.append([id, description, name, aggregate, source])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return [None, None, None, None, None]
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_Search')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getavailableboundaries', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
if result:
resp = []
for element in result:
id = element['boundary_id']
description = element['description']
tspan = element['time_span']
tablename = element['tablename']
resp.append([id, description, tspan, tablename])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

402
server/extension/sql/120_data_observatory_geometries.sql
View File

@ -1,402 +0,0 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundary', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundary')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundaryid', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryId')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
username TEXT,
orgname TEXT,
geometry_id TEXT,
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
username TEXT,
orgname TEXT,
geometry_id TEXT,
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundarybyid', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundariesbygeometry', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundariesbypointandradius', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getpointsbygeometry', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getpointsbypointandradius', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

36
server/extension/sql/125_data_observatory_table_augment.sql
View File

@ -1,36 +0,0 @@
CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text);
CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_server.ds_return_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_GetReturnMetadata;
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS SETOF record AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_FetchJoinFdwTableData;
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
RETURNS boolean AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_DisconnectUserTable;
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;

93
server/extension/sql/130_data_observatory_helper.sql
View File

@ -1,93 +0,0 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text)
RETURNS text AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.obs_dumpversion();
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them
CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators(
username TEXT,
orgname TEXT,
bounds geometry(Geometry, 4326) DEFAULT NULL,
filter_tags TEXT[] DEFAULT NULL,
denom_id TEXT DEFAULT NULL,
geom_id TEXT DEFAULT NULL,
timespan TEXT DEFAULT NULL)
RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators(
username TEXT,
orgname TEXT,
bounds geometry(Geometry, 4326) DEFAULT NULL,
section_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
other_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
ids TEXT[] DEFAULT ARRAY[]::TEXT[],
name TEXT DEFAULT NULL,
denom_id TEXT DEFAULT '',
geom_id TEXT DEFAULT '',
timespan TEXT DEFAULT '')
RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators(
username TEXT,
orgname TEXT,
bounds geometry(Geometry, 4326) DEFAULT NULL,
filter_tags TEXT[] DEFAULT NULL,
numer_id TEXT DEFAULT NULL,
geom_id TEXT DEFAULT NULL,
timespan TEXT DEFAULT NULL)
RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries(
username TEXT,
orgname TEXT,
bounds geometry(Geometry, 4326) DEFAULT NULL,
filter_tags TEXT[] DEFAULT NULL,
numer_id TEXT DEFAULT NULL,
denom_id TEXT DEFAULT NULL,
timespan TEXT DEFAULT NULL,
number_geometries INTEGER DEFAULT NULL)
RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans(
username TEXT,
orgname TEXT,
bounds geometry(Geometry, 4326) DEFAULT NULL,
filter_tags TEXT[] DEFAULT NULL,
numer_id TEXT DEFAULT NULL,
denom_id TEXT DEFAULT NULL,
geom_id TEXT DEFAULT NULL)
RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata(
username TEXT,
orgname TEXT,
aggregate_type TEXT DEFAULT NULL)
RETURNS TABLE(name TEXT, subsection JSON) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;

14
server/extension/sql/15_config_helper.sql
View File

@ -89,17 +89,3 @@ RETURNS boolean AS $$
GD[cache_key] = routing_config GD[cache_key] = routing_config
return True return True
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER; $$ LANGUAGE @@plpythonu@@ SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_obs_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import ObservatoryConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = obs_config
return True
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

12
server/extension/sql/200_quotas.sql
View File

@ -93,18 +93,6 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
provider = user_routing_config.provider provider = user_routing_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider]] ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Observatory
service = 'observatory'
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
user_service = UserMetricsService(user_obs_config, redis_conn)
monthly_quota = user_obs_config.monthly_quota
used_quota = user_service.used_quota(user_obs_config.service_type, today)
soft_limit = user_obs_config.soft_limit
provider = user_obs_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
return ret return ret
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED; $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

44
server/extension/test/expected/100_data_observatory_tables_test.out
View File

@ -1,44 +0,0 @@
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_connectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_getreturnmetadata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_fetchjoinfdwtabledata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, json');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_disconnectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text');
exists
--------
t
(1 row)

287
server/extension/test/expected/100_data_observatory_test.out
View File

@ -1,287 +0,0 @@
\set ECHO none
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdemographicsnapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getsegmentsnapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeasure'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeasurebyid'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getcategory'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getuscensusmeasure'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getuscensuscategory'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getpopulation'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_search'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailableboundaries'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundary'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundaryid'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundarybyid'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeta'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, text[], json');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundariesbygeometry'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundariesbypointandradius'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getpointsbygeometry'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getpointsbypointandradius'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailablenumerators'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailabledenominators'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailablegeometries'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text, integer');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailabletimespans'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_legacybuildermetadata'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_metadatavalidation'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, json, integer');
exists
--------
t
(1 row)

28
server/extension/test/sql/100_data_observatory_tables_test.sql
View File

@ -1,28 +0,0 @@
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_connectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_getreturnmetadata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_fetchjoinfdwtabledata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, json');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_dst_disconnectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text');

188
server/extension/test/sql/100_data_observatory_test.sql
View File

@ -1,188 +0,0 @@
\set ECHO none
-- add the schema cdb_dataservices_server to the SEARCH_PATH
DO $$ BEGIN
PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false);
END $$;
\set ECHO all
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdemographicsnapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getsegmentsnapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeasure'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeasurebyid'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getcategory'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getuscensusmeasure'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getuscensuscategory'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getpopulation'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_search'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailableboundaries'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundary'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundaryid'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundarybyid'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeta'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, text[], json');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundariesbygeometry'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getboundariesbypointandradius'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getpointsbygeometry'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getpointsbypointandradius'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailablenumerators'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailabledenominators'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailablegeometries'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text, integer');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getavailabletimespans'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text[], text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_legacybuildermetadata'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_metadatavalidation'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, json, integer');

55
server/extension/test/sql/366_empty_table_test.sql.template
View File

@ -9,62 +9,7 @@ END $$;
-- Set configuration for a user 'foo' -- Set configuration for a user 'foo'
DO $$ DO $$
import json
from cartodb_services.config import ServiceConfiguration
import cartodb_services import cartodb_services
cartodb_services.init(plpy, GD) cartodb_services.init(plpy, GD)
service_config = ServiceConfiguration('observatory', 'foo', None)
service_config.user.set('soft_obs_general_limit', True)
service_config.user.set('period_end_date', '20170516')
$$ LANGUAGE @@plpythonu@@; $$ LANGUAGE @@plpythonu@@;
-- Mock Observatory backend function
CREATE SCHEMA cdb_observatory;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetData(geomvals geomval[], params JSON, merge BOOLEAN DEFAULT TRUE)
RETURNS TABLE (
id INT,
data JSON
) AS $$
BEGIN
-- this will return an empty set
RAISE NOTICE 'Mocked OBS_GetData()';
END;
$$ LANGUAGE plpgsql;
GRANT USAGE ON SCHEMA cdb_observatory TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api;
-- Test it
SELECT * FROM cdb_dataservices_server.OBS_GetData(
'foo',
NULL,
'{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::_geomval,
'[{"id": 1, "score": 52.7515548093083898758340051256007949661290516400338, "geom_id": "us.census.tiger.census_tract", "denom_id": "us.census.acs.B01003001", "numer_id": "us.census.acs.B03002003", "geom_name": "US Census Tracts", "geom_type": "Geometry", "num_geoms": 2.86483076549783307739486952736, "denom_name": "Total Population", "denom_type": "Numeric", "numer_name": "White Population", "numer_type": "Numeric", "score_rank": 1, "target_area": 0.000307374806576033, "geom_colname": "the_geom", "score_rownum": 1, "target_geoms": null, "denom_colname": "total_pop", "denom_reltype": '
'"denominator", "geom_timespan": "2015", "normalization": "prenormalized", "numer_colname": "white_pop", "timespan_rank": 1, "geom_tablename": "obs_87a814e485deabe3b12545a537f693d16ca702c2", "max_score_rank": null, "numer_timespan": "2010 - 2014", "suggested_name": "white_pop_2010_2014", "denom_aggregate": "sum", "denom_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "numer_aggregate": "sum", "numer_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "timespan_rownum": 1, "geom_description": "Census tracts are small, relatively permanent statistical subdivisions of a county or equivalent entity that are updated by local participants prior to each decennial census as part of the Census Bureau’s Participant Statistical Areas Program. The Census Bureau delineates census tracts in situations where no local participant existed or where state, local, or tribal governments'
'declined to participate. The primary purpose of census tracts is to provide a stable set of geographic units for the presentation of statistical data.\r\n\r\nCensus tracts generally have a population size between 1,200 and 8,000 people, with an optimum size of 4,000 people. A census tract usually covers a contiguous area; however, the spatial size of census tracts varies widely depending on the density of settlement. Census tract boundaries are delineated with the intention of being maintained over a long time so that statistical comparisons can be made from census to census. Census tracts occasionally are split due to population growth or merged as a result of substantial population decline.\r\n\r\nCensus tract boundaries generally follow visible and identifiable features. They may follow nonvisible legal boundaries, such as minor civil division (MCD) or incorporated place boundaries'
'in some states and situations, to allow for census-tract-to-governmental-unit relationships where the governmental boundaries tend to remain unchanged between censuses. State and county boundaries always are census tract boundaries in the standard census geographic hierarchy. Tribal census tracts are a unique geographic entity defined within federally recognized American Indian reservations and off-reservation trust lands and can cross state and county boundaries. Tribal census tracts may be completely different from the census tracts and block groups defined by state and county (see “Tribal Census Tract”).", "denom_description": "The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.", "max_timespan_rank": null, "numer_description": "The number of people identifying as white, non-Hispanic in each'
'geography.", "geom_t_description": null, "denom_t_description": null, "numer_t_description": null, "geom_geomref_colname": "geoid", "denom_geomref_colname": "geoid", "numer_geomref_colname": "geoid"}]'::json,
true);
-- Mock another observatory backend function (overloaded, different params)
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetData(geomrefs TEXT[], params JSON)
RETURNS TABLE (
id INT,
data JSON
) AS $$
BEGIN
-- this will return an empty set
RAISE NOTICE 'Mocked OBS_GetData()';
END;
$$ LANGUAGE plpgsql;
GRANT USAGE ON SCHEMA cdb_observatory TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api;
-- Test it
SELECT * FROM cdb_dataservices_server.OBS_GetData('foo', NULL, '{bar, baz}'::TEXT[], '[]'::JSON);